diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 6302b547982118..b5b2765e43844f 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,7 +1,4 @@ -variables: - coverage: false - -trigger: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] +trigger: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7'] jobs: - job: Prebuild @@ -14,84 +11,6 @@ jobs: - template: ./prebuild-checks.yml -- job: Docs_PR - displayName: Docs PR - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./docs-steps.yml - parameters: - upload: true - - -- job: macOS_CI_Tests - displayName: macOS CI Tests - dependsOn: Prebuild - #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - # bpo-39837: macOS tests on Azure Pipelines are disabled - condition: false - - variables: - testRunTitle: '$(build.sourceBranchName)-macos' - testRunPlatform: macos - - pool: - vmImage: macos-10.15 - - steps: - - template: ./macos-steps.yml - - -- job: Ubuntu_CI_Tests - displayName: Ubuntu CI Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(build.sourceBranchName)-linux' - testRunPlatform: linux - openssl_version: 1.1.1t - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - - -- job: Ubuntu_Coverage_CI_Tests - displayName: Ubuntu CI Tests (coverage) - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['coverage'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(Build.SourceBranchName)-linux-coverage' - testRunPlatform: linux-coverage - openssl_version: 1.1.1t - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - coverage: true - - - job: Windows_CI_Tests displayName: Windows CI Tests dependsOn: Prebuild diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml deleted file mode 100644 index 647daff7a033a8..00000000000000 --- a/.azure-pipelines/docs-steps.yml +++ /dev/null @@ -1,47 +0,0 @@ -parameters: - latex: false - upload: false - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' - inputs: - versionSpec: '>=3.6' - -- script: python -m pip install -r requirements.txt - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Install build dependencies' - -- ${{ if ne(parameters.latex, 'true') }}: - - script: make check html PYTHON=python - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - -- ${{ if eq(parameters.latex, 'true') }}: - - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full - displayName: 'Install LaTeX' - - - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - -- ${{ if eq(parameters.upload, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: 'Publish docs' - - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/build' - ArtifactName: docs - publishLocation: Container - - - ${{ if eq(parameters.latex, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: 'Publish dist' - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/dist' - ArtifactName: docs_dist - publishLocation: Container diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml deleted file mode 100644 index fa38a0df8c87b8..00000000000000 --- a/.azure-pipelines/macos-steps.yml +++ /dev/null @@ -1,27 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev - displayName: 'Configure CPython (debug)' - -- script: make -j4 - displayName: 'Build CPython' - -- script: make pythoninfo - displayName: 'Display build info' - -- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - continueOnError: true - timeoutInMinutes: 30 - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml index 9d7c5e1279f46d..e23c7b1dcb55c1 100644 --- a/.azure-pipelines/posix-steps.yml +++ b/.azure-pipelines/posix-steps.yml @@ -1,10 +1,3 @@ -parameters: - coverage: false - sudo_dependencies: sudo - dependencies: apt - patchcheck: true - xvfb: true - steps: - checkout: self clean: true @@ -14,7 +7,7 @@ steps: - script: sudo setfacl -Rb /home/vsts displayName: 'Workaround ACL issue' -- script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version) +- script: sudo ./.azure-pipelines/posix-deps-apt.sh $(openssl_version) displayName: 'Install dependencies' - script: ./configure --with-pydebug @@ -23,61 +16,11 @@ steps: - script: make -j4 displayName: 'Build CPython' -- ${{ if eq(parameters.coverage, 'true') }}: - - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage - displayName: 'Set up virtual environment' - - - script: ./venv/bin/python -m test.pythoninfo - displayName: 'Display build info' - - - script: | - $COMMAND -m coverage run --pylib -m test \ - --fail-env-changed \ - -uall,-cpu \ - --junit-xml=$(build.binariesDirectory)/test-results.xml \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures - displayName: 'Tests with coverage' - env: - ${{ if eq(parameters.xvfb, 'true') }}: - COMMAND: xvfb-run ./venv/bin/python - ${{ if ne(parameters.xvfb, 'true') }}: - COMMAND: ./venv/bin/python - - - script: ./venv/bin/python -m coverage xml - displayName: 'Generate coverage.xml' - - - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml - displayName: 'Publish code coverage results' - - -- ${{ if ne(parameters.coverage, 'true') }}: - - script: make pythoninfo - displayName: 'Display build info' - - - script: $COMMAND buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - env: - ${{ if eq(parameters.xvfb, 'true') }}: - COMMAND: xvfb-run make - ${{ if ne(parameters.xvfb, 'true') }}: - COMMAND: make - -- ${{ if eq(parameters.patchcheck, 'true') }}: - - script: | - git fetch origin - ./python Tools/patchcheck/patchcheck.py --ci true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - +- script: make pythoninfo + displayName: 'Display build info' -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() +- script: | + git fetch origin + ./python Tools/patchcheck/patchcheck.py --ci true + displayName: 'Run patchcheck.py' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 5f7218768c18af..335a4b407cb83c 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,7 +1,4 @@ -variables: - coverage: false - -pr: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] +pr: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7'] jobs: - job: Prebuild @@ -14,40 +11,8 @@ jobs: - template: ./prebuild-checks.yml -- job: Docs_PR - displayName: Docs PR - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./docs-steps.yml - - -- job: macOS_PR_Tests - displayName: macOS PR Tests - dependsOn: Prebuild - #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - # bpo-39837: macOS tests on Azure Pipelines are disabled - condition: false - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-macos' - testRunPlatform: macos - - pool: - vmImage: macos-10.15 - - steps: - - template: ./macos-steps.yml - parameters: - targetBranch: $(System.PullRequest.TargetBranch) - - -- job: Ubuntu_PR_Tests - displayName: Ubuntu PR Tests +- job: Ubuntu_Patchcheck + displayName: Ubuntu patchcheck dependsOn: Prebuild condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) @@ -57,67 +22,7 @@ jobs: variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.1t - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - - -- job: Ubuntu_Coverage_PR_Tests - displayName: Ubuntu PR Tests (coverage) - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['coverage'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(Build.SourceBranchName)-linux-coverage' - testRunPlatform: linux-coverage - openssl_version: 1.1.1t + openssl_version: 1.1.1u steps: - template: ./posix-steps.yml - parameters: - dependencies: apt - coverage: true - - -- job: Windows_PR_Tests - displayName: Windows PR Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: windows-2022 - - strategy: - matrix: - win32: - arch: win32 - buildOpt: '-p Win32' - testRunTitle: '$(System.PullRequest.TargetBranch)-win32' - testRunPlatform: win32 - win64: - arch: amd64 - buildOpt: '-p x64' - testRunTitle: '$(System.PullRequest.TargetBranch)-win64' - testRunPlatform: win64 - winarm64: - arch: arm64 - buildOpt: '-p arm64' - maxParallel: 4 - - steps: - - template: ./windows-steps.yml - parameters: - targetBranch: $(System.PullRequest.TargetBranch) diff --git a/.azure-pipelines/prebuild-checks.yml b/.azure-pipelines/prebuild-checks.yml index 30ff642d1267a1..2c6460d2386735 100644 --- a/.azure-pipelines/prebuild-checks.yml +++ b/.azure-pipelines/prebuild-checks.yml @@ -11,18 +11,6 @@ steps: displayName: Fetch comparison tree condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) -- script: | - if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)' - then - echo "No docs were updated: docs.run=false" - echo "##vso[task.setvariable variable=run;isOutput=true]false" - else - echo "Docs were updated: docs.run=true" - echo "##vso[task.setvariable variable=run;isOutput=true]true" - fi - displayName: Detect documentation changes - name: docs - - script: | if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)' then diff --git a/.cirrus-DISABLED.yml b/.cirrus-DISABLED.yml new file mode 100644 index 00000000000000..f20835cb6cac2a --- /dev/null +++ b/.cirrus-DISABLED.yml @@ -0,0 +1,29 @@ +# gh-91960: Job disabled since Python is out of free credit (September 2023): +# https://discuss.python.org/t/freebsd-gets-a-new-cirrus-ci-github-action-job-and-a-new-buildbot/33122/26 + +freebsd_task: + freebsd_instance: + matrix: + - image: freebsd-13-2-release-amd64 + # Turn off TCP and UDP blackhole. It is not enabled by default in FreeBSD, + # but it is in the FreeBSD GCE images as used by Cirrus-CI. It causes even + # local local connections to fail with ETIMEDOUT instead of ECONNREFUSED. + # For more information see https://reviews.freebsd.org/D41751 and + # https://github.com/cirruslabs/cirrus-ci-docs/issues/483. + sysctl_script: + - sysctl net.inet.tcp.blackhole=0 + - sysctl net.inet.udp.blackhole=0 + configure_script: + - mkdir build + - cd build + - ../configure --with-pydebug + build_script: + - cd build + - make -j$(sysctl -n hw.ncpu) + pythoninfo_script: + - cd build + - make pythoninfo + test_script: + - cd build + # dtrace fails to build on FreeBSD - see gh-73263 + - make buildbottest TESTOPTS="-j0 -x test_dtrace --timeout=600" diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000000000..b5d94317e8aa8b --- /dev/null +++ b/.coveragerc @@ -0,0 +1,24 @@ +[run] +branch = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + raise AssertionError\( + + # Empty bodies in protocols or abstract methods + ^\s*def [a-zA-Z0-9_]+\(.*\)(\s*->.*)?:\s*\.\.\.(\s*#.*)?$ + ^\s*\.\.\.(\s*#.*)?$ + + .*# pragma: no cover + .*# pragma: no branch + + # Additions for IDLE: + .*# htest # + if not (_htest or _utest): + if not .*_utest: + if .*_htest: + diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index efbdcd402cdf67..9f808af38e69df 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,11 +2,11 @@ FROM docker.io/library/fedora:37 ENV CC=clang -ENV WASI_SDK_VERSION=19 +ENV WASI_SDK_VERSION=20 ENV WASI_SDK_PATH=/opt/wasi-sdk ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=7.0.0 +ENV WASMTIME_VERSION=14.0.4 ENV WASMTIME_CPU_ARCH=x86_64 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9fbaf7fddd8514..0dc303015df5c7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -9,7 +9,9 @@ "-y", "which", "zsh", - "fish" + "fish", + // For umask fix below. + "/usr/bin/setfacl" ], "updateContentCommand": { // Using the shell for `nproc` usage. @@ -22,6 +24,11 @@ "html" ] }, + "postCreateCommand": { + // https://github.com/orgs/community/discussions/26026 + "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."], + "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"] + }, "customizations": { "vscode": { "extensions": [ @@ -31,10 +38,10 @@ "maelvalais.autoconf", // C auto-complete. "ms-vscode.cpptools", - // To view built docs. - "ms-vscode.live-server" - // https://github.com/microsoft/vscode-python/issues/18073 - // "ms-python.python" + // To view HTML build of docs. + "ms-vscode.live-server", + // Python auto-complete. + "ms-python.python" ], "settings": { "C_Cpp.default.compilerPath": "/usr/bin/clang", diff --git a/.editorconfig b/.editorconfig index 81445d2d79c739..0169eed951cd3f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,5 +8,8 @@ indent_style = space [*.{py,c,cpp,h}] indent_size = 4 +[*.rst] +indent_size = 3 + [*.yml] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index 4ed95069442f3d..acfd62411542f1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -18,14 +18,13 @@ *.zip binary # Specific binary files -Lib/test/sndhdrdata/sndhdr.* binary PC/classicAppCompat.* binary # Text files that should not be subject to eol conversion [attr]noeol -text Lib/test/cjkencodings/* noeol -Lib/test/coding20731.py noeol +Lib/test/tokenizedata/coding20731.py noeol Lib/test/decimaltestdata/*.decTest noeol Lib/test/test_email/data/*.txt noeol Lib/test/test_importlib/resources/data01/* noeol @@ -67,26 +66,34 @@ PCbuild/readme.txt dos [attr]generated linguist-generated=true diff=generated **/clinic/*.c.h generated +**/clinic/*.cpp.h generated +**/clinic/*.h.h generated *_db.h generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated +Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/opcode.h generated +Include/opcode_ids.h generated Include/token.h generated +Lib/_opcode_metadata.py generated Lib/keyword.py generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated Lib/token.py generated +Misc/sbom.spdx.json generated Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated Parser/token.c generated Programs/test_frozenmain.h generated Python/Python-ast.c generated +Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/abstract_interp_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d40519e40d3cc2..aa6d937d9cbc31 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,6 +9,7 @@ # pre-commit .pre-commit-config.yaml @hugovk @AlexWaygood +.ruff.toml @hugovk @AlexWaygood # Build system configure* @erlend-aasland @corona10 @@ -22,17 +23,23 @@ configure* @erlend-aasland @corona10 **/*hamt* @1st1 Objects/set* @rhettinger Objects/dict* @methane @markshannon +Objects/typevarobject.c @JelleZijlstra Objects/type* @markshannon Objects/codeobject.c @markshannon Objects/frameobject.c @markshannon Objects/call.c @markshannon -Python/ceval.c @markshannon +Python/ceval*.c @markshannon @gvanrossum +Python/ceval*.h @markshannon @gvanrossum Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel Python/ast_opt.c @isidentical +Python/bytecodes.c @markshannon @gvanrossum +Python/optimizer*.c @markshannon @gvanrossum Lib/test/test_patma.py @brandtbucher Lib/test/test_peepholer.py @brandtbucher +Lib/test/test_type_*.py @JelleZijlstra +Lib/test/test_capi/test_misc.py @markshannon @gvanrossum # Exceptions Lib/traceback.py @iritkatriel @@ -66,7 +73,8 @@ Python/traceback.c @iritkatriel /Tools/build/parse_html5_entities.py @ezio-melotti # Import (including importlib). -**/*import* @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw +**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw +/Python/import.c @kumaraditya303 **/*importlib/resources/* @jaraco @warsaw @FFY00 **/importlib/metadata/* @jaraco @warsaw @@ -77,7 +85,7 @@ Doc/library/time.rst @pganssle @abalkin Lib/test/test_time.py @pganssle @abalkin Modules/timemodule.c @pganssle @abalkin Python/pytime.c @pganssle @abalkin -Include/pytime.h @pganssle @abalkin +Include/internal/pycore_time.h @pganssle @abalkin # Email and related **/*mail* @python/email-team @@ -95,6 +103,11 @@ Include/pytime.h @pganssle @abalkin /Tools/peg_generator/ @pablogsal @lysnikolaou /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou /Grammar/python.gram @pablogsal @lysnikolaou +/Lib/tokenize.py @pablogsal @lysnikolaou +/Lib/test/test_tokenize.py @pablogsal @lysnikolaou + +# Code generator +/Tools/cases_generator/ @gvanrossum # AST Python/ast.c @isidentical @@ -114,6 +127,12 @@ Lib/ast.py @isidentical /Lib/test/test_subprocess.py @gpshead /Modules/*subprocess* @gpshead +# Limited C API & stable ABI +Tools/build/stable_abi.py @encukou +Misc/stable_abi.toml @encukou +Doc/data/*.abi @encukou +Doc/c-api/stable.rst @encukou + # Windows /PC/ @python/windows-team /PCbuild/ @python/windows-team @@ -139,7 +158,7 @@ Lib/ast.py @isidentical **/*idlelib* @terryjreedy -**/*typing* @gvanrossum @Fidget-Spinner @JelleZijlstra @AlexWaygood +**/*typing* @JelleZijlstra @AlexWaygood **/*ftplib @giampaolo **/*shutil @giampaolo @@ -162,4 +181,16 @@ Lib/ast.py @isidentical **/*pathlib* @barneygale # zipfile.Path -**/*zipfile/*_path.py @jaraco +**/*zipfile/_path/* @jaraco + +# Argument Clinic +/Tools/clinic/** @erlend-aasland @AlexWaygood +/Lib/test/test_clinic.py @erlend-aasland @AlexWaygood +Doc/howto/clinic.rst @erlend-aasland + +# WebAssembly +/Tools/wasm/ @brettcannon + +# SBOM +/Misc/sbom.spdx.json @sethmlarson +/Tools/build/generate_sbom.py @sethmlarson diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md deleted file mode 100644 index 1d93e0735e50f3..00000000000000 --- a/.github/ISSUE_TEMPLATE/bug.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Bug report -about: Submit a bug report -labels: "type-bug" ---- - - - -# Bug report - -A clear and concise description of what the bug is. -Include a minimal, reproducible example (https://stackoverflow.com/help/minimal-reproducible-example), if possible. - -# Your environment - - - -- CPython versions tested on: -- Operating system and architecture: - - diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 00000000000000..395516f29b037c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,56 @@ +name: Bug report +description: Submit a bug report +labels: ["type-bug"] +body: + - type: markdown + attributes: + value: | + **New to Python?** + + For help or advice on using Python, try one of the following options instead of opening a GitHub issue: + + - Asking on [Discourse](https://discuss.python.org/c/users/7) or [Stack Overflow](https://stackoverflow.com) + - Reading the [Python tutorial](https://docs.python.org/3/tutorial/) + - Emailing [python-list](https://mail.python.org/mailman/listinfo/python-list) + + Make sure to also search the [CPython issue tracker](https://github.com/python/cpython/issues?q=is%3Aissue+sort%3Acreated-desc) to check that the bug has not already been reported. + - type: textarea + attributes: + label: "Bug description:" + description: > + Give a clear and concise description of what happened. + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016), + and put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true + - type: dropdown + attributes: + label: "CPython versions tested on:" + multiple: true + options: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "CPython main branch" + validations: + required: true + - type: dropdown + attributes: + label: "Operating systems tested on:" + multiple: true + options: + - Linux + - macOS + - Windows + - Other + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/crash.md b/.github/ISSUE_TEMPLATE/crash.md deleted file mode 100644 index dad3423db03410..00000000000000 --- a/.github/ISSUE_TEMPLATE/crash.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: Crash report -about: A hard crash of the interpreter, possibly with a core dump -labels: "type-crash" ---- - - - -# Crash report - -Tell us what happened, ideally including a minimal, reproducible example (https://stackoverflow.com/help/minimal-reproducible-example). - -# Error messages - -Enter any relevant error message caused by the crash, including a core dump if there is one. - -# Your environment - - - -- CPython versions tested on: -- Operating system and architecture: - - diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml new file mode 100644 index 00000000000000..c14d7cf2599d4c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -0,0 +1,54 @@ +name: Crash report +description: A hard crash of the interpreter, possibly with a core dump +labels: ["type-crash"] +body: + - type: markdown + attributes: + value: | + This form is for hard crashes of the Python interpreter, segmentation faults, failed C-level assertions, and similar. Unexpected exceptions raised from Python functions in the standard library count as bugs rather than crashes. + + The CPython interpreter is written in a different programming language, C. A "CPython crash" is when Python itself fails, leading to a traceback in the C stack. + - type: textarea + attributes: + label: What happened? + description: > + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016), + and put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true + - type: dropdown + attributes: + label: "CPython versions tested on:" + multiple: true + options: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "CPython main branch" + validations: + required: true + - type: dropdown + attributes: + label: "Operating systems tested on:" + multiple: true + options: + - Linux + - macOS + - Windows + - Other + validations: + required: false + - type: input + attributes: + label: "Output from running 'python -VV' on the command line:" + description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md deleted file mode 100644 index ed051e945f8120..00000000000000 --- a/.github/ISSUE_TEMPLATE/feature.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: Feature or enhancement -about: Submit a proposal for a new CPython feature or enhancement -labels: "type-feature" ---- - -# Feature or enhancement - -(A clear and concise description of your proposal.) - -# Pitch - -(Explain why this feature or enhancement should be implemented and how it would be used. - Add examples, if applicable.) - -# Previous discussion - - - - - diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 00000000000000..4361ab2bf827fb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,40 @@ +name: Feature or enhancement +description: Submit a proposal for a new CPython feature or enhancement +labels: ["type-feature"] +body: + - type: markdown + attributes: + value: | + # Proposing a feature to CPython? + + You'll need to demonstrate widespread support for your idea among the community. + + Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form. + - type: textarea + attributes: + label: "Proposal:" + description: > + Explain your proposal, why it should be implemented, and how it would be used. + Add examples, if applicable. + Put any code blocks inside triple backticks. + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true + - type: dropdown + attributes: + label: Has this already been discussed elsewhere? + options: + - No response given + - I have already discussed this feature proposal on Discourse + - This is a minor feature, which does not need previous discussion elsewhere + multiple: false + validations: + required: true + - type: textarea + attributes: + label: "Links to previous discussion of this feature:" + validations: + required: false diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f026b0f5f9454a..c8a3165d690364 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,7 +13,7 @@ updates: - "version-update:semver-minor" - "version-update:semver-patch" - package-ecosystem: "pip" - directory: "/Tools/clinic/" + directory: "/Tools/" schedule: interval: "monthly" labels: diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json deleted file mode 100644 index 09848608a7b034..00000000000000 --- a/.github/problem-matchers/sphinx.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "sphinx-problem-matcher", - "pattern": [ - { - "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "line": 2, - "severity": 3, - "message": 4 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose", - "pattern": [ - { - "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst", - "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "severity": 2, - "message": 3 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose-no-severity", - "pattern": [ - { - "_comment": "Looks for file names ending with .rst and line numbers but without severity", - "regexp": "^(.*\\.rst):(\\d+):(.*)$", - "file": 1, - "line": 2, - "message": 3 - } - ] - } - ] -} \ No newline at end of file diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml new file mode 100644 index 00000000000000..570b8779994a0f --- /dev/null +++ b/.github/workflows/add-issue-header.yml @@ -0,0 +1,53 @@ +name: Add issue header +# Automatically edits an issue's descriptions with a header, +# one of: +# +# - Bug report +# - Crash report +# - Feature or enhancement + +on: + issues: + types: + # Only ever run once + - opened + + +jobs: + add-header: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - uses: actions/github-script@v7 + with: + # language=JavaScript + script: | + // https://devguide.python.org/triage/labels/#type-labels + const HEADERS = new Map([ + ['type-bug', 'Bug report'], + ['type-crash', 'Crash report'], + ['type-feature', 'Feature or enhancement'], + ]); + let issue_data = await github.rest.issues.get({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo + }).then(issue => issue.data); + let header = ''; + for (const label_data of issue_data.labels) { + const label_name = (typeof label_data === 'string') ? label_data : label_data.name; + if (HEADERS.has(label_name)) { + header = HEADERS.get(label_name); + break; + } + } + if (header !== '') { + console.log(`Setting new header: ${header}`); + await github.rest.issues.update({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `# ${header}\n\n${issue_data.body.replaceAll('\r', '')}` + }); + } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69b78e5567adb1..cfb36c8c32e18d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,25 +8,25 @@ on: push: branches: - 'main' + - '3.12' - '3.11' - '3.10' - '3.9' - '3.8' - - '3.7' pull_request: branches: - 'main' + - '3.12' - '3.11' - '3.10' - '3.9' - '3.8' - - '3.7' permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable cancel-in-progress: true jobs: @@ -35,10 +35,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: + run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} run_tests: ${{ steps.check.outputs.run_tests }} run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} + run_cifuzz: ${{ steps.check.outputs.run_cifuzz }} + config_hash: ${{ steps.config_hash.outputs.hash }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check for source changes id: check run: | @@ -59,7 +62,7 @@ jobs: # into the PR branch anyway. # # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true fi # Check if we should run hypothesis tests @@ -73,38 +76,87 @@ jobs: echo "run_hypothesis=true" >> $GITHUB_OUTPUT fi + # oss-fuzz maintains a configuration for fuzzing the main branch of + # CPython, so CIFuzz should be run only for code that is likely to be + # merged into the main branch; compatibility with older branches may + # be broken. + FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' + if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then + # The tests are pretty slow so they are executed only for PRs + # changing relevant files. + echo "Run CIFuzz tests" + echo "run_cifuzz=true" >> $GITHUB_OUTPUT + else + echo "Branch too old for CIFuzz tests; or no C files were changed" + echo "run_cifuzz=false" >> $GITHUB_OUTPUT + fi + - name: Compute hash for config cache key + id: config_hash + run: | + echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT + - name: Get a list of the changed documentation-related files + if: github.event_name == 'pull_request' + id: changed-docs-files + uses: Ana06/get-changed-files@v2.2.0 + with: + filter: | + Doc/** + Misc/** + .github/workflows/reusable-docs.yml + format: csv # works for paths with spaces + - name: Check for docs changes + if: >- + github.event_name == 'pull_request' + && steps.changed-docs-files.outputs.added_modified_renamed != '' + id: docs-changes + run: | + echo "run-docs=true" >> "${GITHUB_OUTPUT}" + + check-docs: + name: Docs + needs: check_source + if: fromJSON(needs.check_source.outputs.run-docs) + uses: ./.github/workflows/reusable-docs.yml + check_generated_files: name: 'Check if generated files are up to date' - runs-on: ubuntu-latest + # Don't use ubuntu-latest but a specific version to make the job + # reproducible: to get the same tools versions (autoconf, aclocal, ...) + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 - - name: Check Autoconf version 2.69 and aclocal 1.16.3 + - name: Check Autoconf and aclocal versions run: | - grep "Generated by GNU Autoconf 2.69" configure - grep "aclocal 1.16.3" aclocal.m4 + grep "Generated by GNU Autoconf 2.71" configure + grep "aclocal 1.16.5" aclocal.m4 grep -q "runstatedir" configure grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 - name: Configure CPython run: | # Build Python with the libpython dynamic library - ./configure --with-pydebug --enable-shared - - name: Regenerate autoconf files with container image - run: make regen-configure + ./configure --config-cache --with-pydebug --enable-shared + - name: Regenerate autoconf files + # Same command used by Tools/build/regen-configure.sh ($AUTORECONF) + run: autoreconf -ivf -Werror - name: Build CPython run: | - # Deepfreeze will usually cause global objects to be added or removed, - # so we run it before regen-global-objects gets rum (in regen-all). - make regen-deepfreeze make -j4 regen-all make regen-stdlib-module-names - name: Check for changes @@ -129,130 +181,63 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals - build_win32: - name: 'Windows (x86)' - runs-on: windows-latest - timeout-minutes: 60 + build_windows: + name: 'Windows' needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v3 - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p Win32 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + uses: ./.github/workflows/reusable-windows.yml - build_win_amd64: - name: 'Windows (x64)' - runs-on: windows-latest - timeout-minutes: 60 + build_windows_free_threaded: + name: 'Windows (free-threaded)' needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v3 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p x64 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + uses: ./.github/workflows/reusable-windows.yml + with: + free-threaded: true build_macos: name: 'macOS' - runs-on: macos-latest - timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - HOMEBREW_NO_ANALYTICS: 1 - HOMEBREW_NO_AUTO_UPDATE: 1 - HOMEBREW_NO_INSTALL_CLEANUP: 1 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v3 - - name: Install Homebrew dependencies - run: brew install pkg-config openssl@1.1 xz gdbm tcl-tk - - name: Configure CPython - run: | - CFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ - LDFLAGS="-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" \ - PKG_CONFIG_PATH="$(brew --prefix tcl-tk)/lib/pkgconfig" \ - ./configure \ - --with-pydebug \ - --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@1.1)" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: make buildbottest TESTOPTS="-j4 -uall,-cpu" + uses: ./.github/workflows/reusable-macos.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + + build_macos_free_threaded: + name: 'macOS (free-threaded)' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-macos.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + free-threaded: true build_ubuntu: name: 'Ubuntu' - runs-on: ubuntu-20.04 - timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' - env: - OPENSSL_VER: 1.1.1t - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v3 - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: Tests - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + uses: ./.github/workflows/reusable-ubuntu.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --with-openssl=$OPENSSL_DIR + + build_ubuntu_free_threaded: + name: 'Ubuntu (free-threaded)' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-ubuntu.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + options: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --with-openssl=$OPENSSL_DIR \ + --disable-gil build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' @@ -263,14 +248,19 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1t, 3.0.8, 3.1.0-beta1] + openssl_ver: [1.1.1w, 3.0.11, 3.1.3] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -295,7 +285,7 @@ jobs: - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 - name: Configure CPython - run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR + run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR - name: Build CPython run: make -j4 - name: Display build info @@ -304,16 +294,16 @@ jobs: run: ./python Lib/test/ssltests.py test_hypothesis: - name: "Hypothesis Tests on Ubuntu" + name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-20.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 1.1.1t + OPENSSL_VER: 3.0.11 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -345,9 +335,18 @@ jobs: run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR + run: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --with-openssl=$OPENSSL_DIR - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 @@ -367,7 +366,15 @@ jobs: VENV_PYTHON=$VENV_LOC/bin/python echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV - ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis + ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt + - name: 'Restore Hypothesis database' + id: cache-hypothesis-database + uses: actions/cache@v3 + with: + path: ./hypothesis + key: hypothesis-database-${{ github.head_ref || github.run_id }} + restore-keys: | + - hypothesis-database- - name: "Run tests" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -388,6 +395,11 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig + - uses: actions/upload-artifact@v3 + if: always() + with: + name: hypothesis-example-db + path: .hypothesis/examples/ build_asan: @@ -397,11 +409,16 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1t + OPENSSL_VER: 3.0.11 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -430,10 +447,118 @@ jobs: - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 - name: Configure CPython - run: ./configure --with-address-sanitizer --without-pymalloc + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython run: make -j4 - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: xvfb-run make test + + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ + cifuzz: + name: CIFuzz + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_cifuzz == 'true' + permissions: + security-events: write + strategy: + fail-fast: false + matrix: + sanitizer: [address, undefined, memory] + steps: + - name: Build fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: cpython3 + sanitizer: ${{ matrix.sanitizer }} + - name: Run fuzzers (${{ matrix.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: cpython3 + output-sarif: true + sanitizer: ${{ matrix.sanitizer }} + - name: Upload crash + uses: actions/upload-artifact@v3 + if: failure() && steps.build.outcome == 'success' + with: + name: ${{ matrix.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif + + all-required-green: # This job does nothing and is only used for the branch protection + name: All required checks pass + if: always() + + needs: + - check_source # Transitive dependency, needed to access `run_tests` value + - check-docs + - check_generated_files + - build_macos + - build_macos_free_threaded + - build_ubuntu + - build_ubuntu_free_threaded + - build_ubuntu_ssltests + - build_windows + - build_windows_free_threaded + - test_hypothesis + - build_asan + - cifuzz + + runs-on: ubuntu-latest + + steps: + - name: Check whether the needed jobs succeeded or failed + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe + with: + allowed-failures: >- + build_ubuntu_ssltests, + cifuzz, + test_hypothesis, + allowed-skips: >- + ${{ + !fromJSON(needs.check_source.outputs.run-docs) + && ' + check-docs, + ' + || '' + }} + ${{ + needs.check_source.outputs.run_tests != 'true' + && ' + check_generated_files, + build_macos, + build_macos_free_threaded, + build_ubuntu, + build_ubuntu_free_threaded, + build_ubuntu_ssltests, + build_windows, + build_windows_free_threaded, + build_asan, + ' + || '' + }} + ${{ + !fromJSON(needs.check_source.outputs.run_cifuzz) + && ' + cifuzz, + ' + || '' + }} + ${{ + !fromJSON(needs.check_source.outputs.run_hypothesis) + && ' + test_hypothesis, + ' + || '' + }} + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index 2bed09014e0ff2..29282dffa37ec0 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -8,12 +8,14 @@ on: - '3.*' paths: - 'Tools/msi/**' + - '.github/workflows/build_msi.yml' pull_request: branches: - 'main' - '3.*' paths: - 'Tools/msi/**' + - '.github/workflows/build_msi.yml' permissions: contents: read @@ -31,6 +33,6 @@ jobs: matrix: type: [x86, x64, arm64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build CPython installer - run: .\Tools\msi\build.bat -${{ matrix.type }} + run: .\Tools\msi\build.bat --doc -${{ matrix.type }} diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml deleted file mode 100644 index 3f7550cc72943b..00000000000000 --- a/.github/workflows/doc.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: Docs - -on: - workflow_dispatch: - #push: - # branches: - # - 'main' - # - '3.11' - # - '3.10' - # - '3.9' - # - '3.8' - # - '3.7' - # paths: - # - 'Doc/**' - pull_request: - branches: - - 'main' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' - paths: - - 'Doc/**' - - 'Misc/**' - - '.github/workflows/doc.yml' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build_doc: - name: 'Docs' - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - - name: 'Set up Python' - uses: actions/setup-python@v4 - with: - python-version: '3' - cache: 'pip' - cache-dependency-path: 'Doc/requirements.txt' - - name: 'Install build dependencies' - run: make -C Doc/ venv - - name: 'Check documentation' - run: make -C Doc/ check - - name: 'Build HTML documentation' - run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - - # Add pull request annotations for Sphinx nitpicks (missing references) - - name: 'Get list of changed files' - if: github.event_name == 'pull_request' - id: changed_files - uses: Ana06/get-changed-files@v2.2.0 - with: - filter: "Doc/**" - - name: 'Build changed files in nit-picky mode' - if: github.event_name == 'pull_request' - continue-on-error: true - run: | - # Mark files the pull request modified - touch ${{ steps.changed_files.outputs.added_modified }} - # Build docs with the '-n' (nit-picky) option; convert warnings to annotations - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n --keep-going" html 2>&1 | - python Doc/tools/warnings-to-gh-actions.py - - # Ensure some files always pass Sphinx nit-picky mode (no missing references) - - name: 'Build known-good files in nit-picky mode' - run: | - # Mark files that must pass nit-picky - python Doc/tools/touch-clean-files.py - # Build docs with the '-n' (nit-picky) option, convert warnings to errors (-W) - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going" html 2>&1 - - # This build doesn't use problem matchers or check annotations - # It also does not run 'make check', as sphinx-lint is not installed into the - # environment. - build_doc_oldest_supported_sphinx: - name: 'Docs (Oldest Sphinx)' - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - name: 'Set up Python' - uses: actions/setup-python@v4 - with: - python-version: '3.11' # known to work with Sphinx 3.2 - cache: 'pip' - cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt' - - name: 'Install build dependencies' - run: make -C Doc/ venv REQUIREMENTS="requirements-oldest-sphinx.txt" - - name: 'Build HTML documentation' - run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - - # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release - doctest: - name: 'Doctest' - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} - restore-keys: | - ubuntu-doc- - - name: 'Install Dependencies' - run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican - - name: 'Configure CPython' - run: ./configure --with-pydebug - - name: 'Build CPython' - run: make -j4 - - name: 'Install build dependencies' - run: make -C Doc/ PYTHON=../python venv - # Use "xvfb-run" since some doctest tests open GUI windows - - name: 'Run documentation doctest' - run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="-W --keep-going" doctest diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4481ea80bfd936..6c1c29a58cf4fc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,6 +5,10 @@ on: [push, pull_request, workflow_dispatch] permissions: contents: read +env: + FORCE_COLOR: 1 + RUFF_OUTPUT_FORMAT: github + concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -15,7 +19,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.x" diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 1315bb5a966f01..792903a90a4880 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -7,8 +7,14 @@ on: - main pull_request: paths: - - "Tools/clinic/**" - ".github/workflows/mypy.yml" + - "Lib/test/libregrtest/**" + - "Tools/build/generate_sbom.py" + - "Tools/cases_generator/**" + - "Tools/clinic/**" + - "Tools/peg_generator/**" + - "Tools/requirements-dev.txt" + - "Tools/wasm/**" workflow_dispatch: permissions: @@ -25,15 +31,25 @@ concurrency: jobs: mypy: - name: Run mypy on Tools/clinic/ + strategy: + matrix: + target: [ + "Lib/test/libregrtest", + "Tools/build/", + "Tools/cases_generator", + "Tools/clinic", + "Tools/peg_generator", + "Tools/wasm", + ] + name: Run mypy on ${{ matrix.target }} runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "3.x" + python-version: "3.11" cache: pip - cache-dependency-path: Tools/clinic/requirements-dev.txt - - run: pip install -r Tools/clinic/requirements-dev.txt - - run: mypy --config-file Tools/clinic/mypy.ini + cache-dependency-path: Tools/requirements-dev.txt + - run: pip install -r Tools/requirements-dev.txt + - run: mypy --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 73806c5d6d58af..9f1a8a824e5f19 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: - node-version: 14 + node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v6 + uses: actions/github-script@v7 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: @@ -41,7 +41,10 @@ jobs: url : issue.data.html_url, labels : issue.data.labels.map(label => { return label.name }).join(", "), assignee : issue.data.assignees.map(assignee => { return assignee.login }), - body : issue.data.body + // We need to truncate the body size, because the max size for + // the whole payload is 16kb. We want to be safe and assume that + // body can take up to ~8kb of space. + body : issue.data.body.substring(0, 8000) }; const data = { diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index a220896f2cd7be..0800401f4cd113 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,9 +1,11 @@ #!/bin/sh apt-get update +# autoconf-archive is needed by autoreconf (check_generated_files job) apt-get -yq install \ build-essential \ pkg-config \ + autoconf-archive \ ccache \ gdb \ lcov \ @@ -19,6 +21,7 @@ apt-get -yq install \ libssl-dev \ lzma \ lzma-dev \ + strace \ tk-dev \ uuid-dev \ xvfb \ diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 88aaea039f04f4..080204bcfd3b94 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -5,8 +5,8 @@ on: types: [opened, reopened, labeled, unlabeled, synchronize] permissions: - issues: read - pull-requests: read + issues: write + pull-requests: write jobs: label: @@ -15,7 +15,7 @@ jobs: timeout-minutes: 10 steps: - - uses: mheap/github-action-required-labels@v4 + - uses: mheap/github-action-required-labels@v5 with: mode: exactly count: 0 diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml new file mode 100644 index 00000000000000..1c4fa4239c1e34 --- /dev/null +++ b/.github/workflows/reusable-docs.yml @@ -0,0 +1,108 @@ +name: Docs + +on: + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build_doc: + name: 'Docs' + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + branch_base: 'origin/${{ github.event.pull_request.base.ref }}' + branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' + refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' + steps: + - name: 'Check out latest PR branch commit' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 + - name: 'Fetch commits to get branch diff' + run: | + # Fetch enough history to find a common ancestor commit (aka merge-base): + git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ + --no-tags --prune --no-recurse-submodules + + # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} ) + DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) + + # Get all commits since that commit date from the base branch (eg: master or main): + git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ + --no-tags --prune --no-recurse-submodules + - name: 'Set up Python' + uses: actions/setup-python@v4 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: make -C Doc/ venv + + # To annotate PRs with Sphinx nitpicks (missing references) + - name: 'Build HTML documentation' + continue-on-error: true + run: | + set -Eeuo pipefail + # Build docs with the '-n' (nit-picky) option; write warnings to file + make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going -w sphinx-warnings.txt" html + - name: 'Check warnings' + if: github.event_name == 'pull_request' + run: | + python Doc/tools/check-warnings.py \ + --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ + --fail-if-regression \ + --fail-if-improved + + # This build doesn't use problem matchers or check annotations + build_doc_oldest_supported_sphinx: + name: 'Docs (Oldest Sphinx)' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: 'Set up Python' + uses: actions/setup-python@v4 + with: + python-version: '3.11' # known to work with Sphinx 4.2 + cache: 'pip' + cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt' + - name: 'Install build dependencies' + run: make -C Doc/ venv REQUIREMENTS="requirements-oldest-sphinx.txt" + - name: 'Build HTML documentation' + run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html + + # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release + doctest: + name: 'Doctest' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} + restore-keys: | + ubuntu-doc- + - name: 'Install Dependencies' + run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican + - name: 'Configure CPython' + run: ./configure --with-pydebug + - name: 'Build CPython' + run: make -j4 + - name: 'Install build dependencies' + run: make -C Doc/ PYTHON=../python venv + # Use "xvfb-run" since some doctest tests open GUI windows + - name: 'Run documentation doctest' + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="-W --keep-going" doctest diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml new file mode 100644 index 00000000000000..22f46d18e1b43a --- /dev/null +++ b/.github/workflows/reusable-macos.yml @@ -0,0 +1,46 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + free-threaded: + required: false + type: boolean + default: false + +jobs: + build_macos: + name: 'build and test' + runs-on: macos-latest + timeout-minutes: 60 + env: + HOMEBREW_NO_ANALYTICS: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v4 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} + - name: Install Homebrew dependencies + run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + - name: Configure CPython + run: | + GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ + GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ + ./configure \ + --config-cache \ + --with-pydebug \ + ${{ inputs.free-threaded && '--disable-gil' || '' }} \ + --prefix=/opt/python-dev \ + --with-openssl="$(brew --prefix openssl@3.0)" + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: make test diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml new file mode 100644 index 00000000000000..819b45bda7f980 --- /dev/null +++ b/.github/workflows/reusable-ubuntu.yml @@ -0,0 +1,71 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + options: + required: true + type: string + +jobs: + build_ubuntu_reusable: + name: 'build and test' + timeout-minutes: 60 + runs-on: ubuntu-20.04 + env: + OPENSSL_VER: 3.0.11 + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v4 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }} + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: ${{ inputs.options }} + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + - name: Tests + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: xvfb-run make test diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml new file mode 100644 index 00000000000000..29e0a7e35b5450 --- /dev/null +++ b/.github/workflows/reusable-windows.yml @@ -0,0 +1,53 @@ +on: + workflow_call: + inputs: + free-threaded: + required: false + type: boolean + default: false + +jobs: + build_win32: + name: 'build and test (x86)' + runs-on: windows-latest + timeout-minutes: 60 + env: + IncludeUwp: 'true' + steps: + - uses: actions/checkout@v4 + - name: Build CPython + run: .\PCbuild\build.bat -e -d -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }} + - name: Display build info + run: .\python.bat -m test.pythoninfo + - name: Tests + run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci + + build_win_amd64: + name: 'build and test (x64)' + runs-on: windows-latest + timeout-minutes: 60 + env: + IncludeUwp: 'true' + steps: + - uses: actions/checkout@v4 + - name: Register MSVC problem matcher + run: echo "::add-matcher::.github/problem-matchers/msvc.json" + - name: Build CPython + run: .\PCbuild\build.bat -e -d -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }} + - name: Display build info + run: .\python.bat -m test.pythoninfo + - name: Tests + run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci + + build_win_arm64: + name: 'build (arm64)' + runs-on: windows-latest + timeout-minutes: 60 + env: + IncludeUwp: 'true' + steps: + - uses: actions/checkout@v4 + - name: Register MSVC problem matcher + run: echo "::add-matcher::.github/problem-matchers/msvc.json" + - name: Build CPython + run: .\PCbuild\build.bat -e -d -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }} diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 17d841f1f1c54a..4a545037bf6e2b 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3' diff --git a/.gitignore b/.gitignore index d9c4a7972f076d..c424a894c2a6e0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,10 @@ *.gc?? *.profclang? *.profraw +# Copies of binaries before BOLT optimizations. +*.prebolt +# BOLT profile data. +*.fdata *.dyn .gdb_history .purify @@ -38,6 +42,7 @@ gmon.out .coverage .mypy_cache/ .pytest_cache/ +.ruff_cache/ .DS_Store *.exe @@ -57,7 +62,6 @@ Doc/.venv/ Doc/env/ Doc/.env/ Include/pydtrace_probes.h -Lib/lib2to3/*.pickle Lib/site-packages/* !Lib/site-packages/README.txt Lib/test/data/* @@ -121,9 +125,11 @@ Tools/unicode/data/ /config.status.lineno # hendrikmuhs/ccache-action@v1 /.ccache +/cross-build/ /platform /profile-clean-stamp /profile-run-stamp +/profile-bolt-stamp /Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 808622f19a3dbf..35d9c64a8c5c15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,36 @@ repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.2 + hooks: + - id: ruff + name: Run Ruff on Lib/test/ + args: [--exit-non-zero-on-fix] + files: ^Lib/test/ + - id: ruff + name: Run Ruff on Argument Clinic + args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] + files: ^Tools/clinic/|Lib/test/test_clinic.py + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: + - id: check-toml + exclude: ^Lib/test/test_tomllib/ - id: check-yaml + - id: end-of-file-fixer + types: [python] + exclude: Lib/test/tokenizedata/coding20731.py - id: trailing-whitespace - types_or: [c, python, rst] + types_or: [c, inc, python, rst] + + - repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v0.8.1 + hooks: + - id: sphinx-lint + args: [--enable=default-role] + files: ^Doc/|^Misc/NEWS.d/next/ + + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/.readthedocs.yml b/.readthedocs.yml index 898a9ae89dbb92..59830c79a404e0 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -13,6 +13,20 @@ build: python: "3" commands: + # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition + # + # Cancel building pull requests when there aren't changes in the Doc directory. + # + # If there are no changes (git diff exits with 0) we force the command to return with 183. + # This is a special exit code on Read the Docs that will cancel the build immediately. + - | + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ]; + then + echo "No changes to Doc/ - exiting the build."; + exit 183; + fi + - make -C Doc venv html - mkdir _readthedocs - mv Doc/build/html _readthedocs/html + diff --git a/Doc/Makefile b/Doc/Makefile index c11ea6ce03e8a4..7af56e965e1be4 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -7,7 +7,6 @@ PYTHON = python3 VENVDIR = ./venv SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build -SPHINXLINT = PATH=$(VENVDIR)/bin:$$PATH sphinx-lint BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb JOBS = auto PAPER = @@ -30,6 +29,7 @@ help: @echo " venv to create a venv with necessary tools" @echo " html to make standalone HTML files" @echo " htmlview to open the index page built by the html target in your browser" + @echo " htmllive to rebuild and reload HTML files in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " text to make plain text files" @@ -140,6 +140,11 @@ pydoc-topics: build htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" +.PHONY: htmllive +htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild +htmllive: SPHINXOPTS = --re-ignore="/venv/" +htmllive: html + .PHONY: clean clean: clean-venv -rm -rf build/* @@ -216,11 +221,9 @@ dist: rm dist/python-$(DISTVERSION)-docs-texinfo.tar .PHONY: check -check: - # Check the docs and NEWS files with sphinx-lint. - # Ignore the tools and venv dirs and check that the default role is not used. - $(SPHINXLINT) -i tools -i $(VENVDIR) --enable default-role - $(SPHINXLINT) --enable default-role ../Misc/NEWS.d/next/ +check: venv + $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit + $(VENVDIR)/bin/python3 -m pre_commit run --all-files .PHONY: serve serve: diff --git a/Doc/bugs.rst b/Doc/bugs.rst index d98192b369603e..908987cf41ff6e 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -38,7 +38,7 @@ though it may take a while to be processed. `Helping with Documentation `_ Comprehensive guide for individuals that are interested in contributing to Python documentation. - `Documentation Translations `_ + `Documentation Translations `_ A list of GitHub pages for documentation translation and their primary contacts. diff --git a/Doc/c-api/abstract.rst b/Doc/c-api/abstract.rst index 1823f9d70c79f3..f5df09fa7fd786 100644 --- a/Doc/c-api/abstract.rst +++ b/Doc/c-api/abstract.rst @@ -24,4 +24,3 @@ but whose items have not been set to some non-\ ``NULL`` value yet. mapping.rst iter.rst buffer.rst - objbuffer.rst diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 0a8fcc5ae5fcdf..b3609c233156b6 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -27,22 +27,26 @@ Allocating Objects on the Heap length information for a variable-size object. -.. c:function:: TYPE* PyObject_New(TYPE, PyTypeObject *type) +.. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *type*. Fields not defined by the Python object header - are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of - the type object. + Allocate a new Python object using the C structure type *TYPE* + and the Python type object *typeobj* (``PyTypeObject*``). + Fields not defined by the Python object header are not initialized. + The caller will own the only reference to the object + (i.e. its reference count will be one). + The size of the memory allocation is determined from the + :c:member:`~PyTypeObject.tp_basicsize` field of the type object. -.. c:function:: TYPE* PyObject_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) +.. c:macro:: PyObject_NewVar(TYPE, typeobj, size) Allocate a new Python object using the C structure type *TYPE* and the - Python type object *type*. Fields not defined by the Python object header + Python type object *typeobj* (``PyTypeObject*``). + Fields not defined by the Python object header are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* fields of the size given by the :c:member:`~PyTypeObject.tp_itemsize` field of - *type*. This is useful for implementing objects like tuples, which are + plus *size* (``Py_ssize_t``) fields of the size + given by the :c:member:`~PyTypeObject.tp_itemsize` field of + *typeobj*. This is useful for implementing objects like tuples, which are able to determine their size at construction time. Embedding the array of fields into the same allocation decreases the number of allocations, improving the memory management efficiency. @@ -50,8 +54,8 @@ Allocating Objects on the Heap .. c:function:: void PyObject_Del(void *op) - Releases memory allocated to an object using :c:func:`PyObject_New` or - :c:func:`PyObject_NewVar`. This is normally called from the + Releases memory allocated to an object using :c:macro:`PyObject_New` or + :c:macro:`PyObject_NewVar`. This is normally called from the :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of the object should not be accessed after this call as the memory is no longer a valid Python object. diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 62d542966622ce..f6c8284daeacb0 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -60,7 +60,7 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. - This version is also available via the symbol :data:`Py_Version`. + This version is also available via the symbol :c:var:`Py_Version`. .. c:var:: const unsigned long Py_Version diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 9713431688d499..834aae9372fe3b 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -27,9 +27,18 @@ unit; the entry in (round) parentheses is the Python object type that matches the format unit; and the entry in [square] brackets is the type of the C variable(s) whose address should be passed. +.. _arg-parsing-string-and-buffers: + Strings and buffers ------------------- +.. note:: + + On Python 3.12 and older, the macro :c:macro:`!PY_SSIZE_T_CLEAN` must be + defined before including :file:`Python.h` to use all ``#`` variants of + formats (``s#``, ``y#``, etc.) explained below. + This is not necessary on Python 3.13 and later. + These formats allow accessing an object as a contiguous chunk of memory. You don't have to provide raw storage for the returned unicode or bytes area. @@ -68,15 +77,6 @@ There are three ways strings and buffers can be converted to C: whether the input object is immutable (e.g. whether it would honor a request for a writable buffer, or whether another thread can mutate the data). -.. note:: - - For all ``#`` variants of formats (``s#``, ``y#``, etc.), the macro - :c:macro:`PY_SSIZE_T_CLEAN` must be defined before including - :file:`Python.h`. On Python 3.9 and older, the type of the length argument - is :c:type:`Py_ssize_t` if the :c:macro:`PY_SSIZE_T_CLEAN` macro is defined, - or int otherwise. - - ``s`` (:class:`str`) [const char \*] Convert a Unicode object to a C pointer to a character string. A pointer to an existing string is stored in the character pointer @@ -293,8 +293,10 @@ Other objects ``O`` (object) [PyObject \*] Store a Python object (without any conversion) in a C object pointer. The C - program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not ``NULL``. + program thus receives the actual object that was passed. A new + :term:`strong reference` to the object is not created + (i.e. its reference count is not increased). + The pointer stored is not ``NULL``. ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but @@ -343,7 +345,7 @@ Other objects *items*. Format units for sequences may be nested. It is possible to pass "long" integers (integers whose value exceeds the -platform's :const:`LONG_MAX`) however no proper range checking is done --- the +platform's :c:macro:`LONG_MAX`) however no proper range checking is done --- the most significant bits are silently truncated when the receiving field is too small to receive the value (actually, the semantics are inherited from downcasts in C --- your mileage may vary). @@ -378,7 +380,8 @@ inside nested parentheses. They are: mutually exclude each other. Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! +*borrowed* references; do not release them +(i.e. do not decrement their reference count)! Additional arguments passed to these functions must be addresses of variables whose type is determined by the format string; these are used to store values @@ -410,21 +413,35 @@ API Functions than a variable number of arguments. -.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...) +.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char * const *keywords, ...) Parse the parameters of a function that takes both positional and keyword - parameters into local variables. The *keywords* argument is a - ``NULL``-terminated array of keyword parameter names. Empty names denote + parameters into local variables. + The *keywords* argument is a ``NULL``-terminated array of keyword parameter + names specified as null-terminated ASCII or UTF-8 encoded C strings. + Empty names denote :ref:`positional-only parameters `. Returns true on success; on failure, it returns false and raises the appropriate exception. + .. note:: + + The *keywords* parameter declaration is :c:expr:`char * const *` in C and + :c:expr:`const char * const *` in C++. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + .. versionchanged:: 3.6 Added support for :ref:`positional-only parameters `. + .. versionchanged:: 3.13 + The *keywords* parameter has now type :c:expr:`char * const *` in C and + :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. + Added support for non-ASCII keyword parameter names. + -.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs) + +.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char * const *keywords, va_list vargs) Identical to :c:func:`PyArg_ParseTupleAndKeywords`, except that it accepts a va_list rather than a variable number of arguments. @@ -439,23 +456,31 @@ API Functions .. versionadded:: 3.2 -.. XXX deprecated, will be removed .. c:function:: int PyArg_Parse(PyObject *args, const char *format, ...) - Function used to deconstruct the argument lists of "old-style" functions --- - these are functions which use the :const:`METH_OLDARGS` parameter parsing - method, which has been removed in Python 3. This is not recommended for use - in parameter parsing in new code, and most code in the standard interpreter - has been modified to no longer use this for that purpose. It does remain a - convenient way to decompose other tuples, however, and may continue to be - used for that purpose. + Parse the parameter of a function that takes a single positional parameter + into a local variable. Returns true on success; on failure, it returns + false and raises the appropriate exception. + + Example:: + + // Function using METH_O calling convention + static PyObject* + my_function(PyObject *module, PyObject *arg) + { + int value; + if (!PyArg_Parse(arg, "i:my_function", &value)) { + return NULL; + } + // ... use value ... + } .. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) A simpler form of parameter retrieval which does not use a format string to specify the types of the arguments. Functions which use this method to retrieve - their parameters should be declared as :const:`METH_VARARGS` in function or + their parameters should be declared as :c:macro:`METH_VARARGS` in function or method tables. The tuple containing the actual parameters should be passed as *args*; it must actually be a tuple. The length of the tuple must be at least *min* and no more than *max*; *min* and *max* may be equal. Additional @@ -469,7 +494,7 @@ API Functions will be set if there was a failure. This is an example of the use of this function, taken from the sources for the - :mod:`_weakref` helper module for weak references:: + :mod:`!_weakref` helper module for weak references:: static PyObject * weakref_ref(PyObject *self, PyObject *args) @@ -489,6 +514,19 @@ API Functions PyArg_ParseTuple(args, "O|O:ref", &object, &callback) +.. c:macro:: PY_CXX_CONST + + The value to be inserted, if any, before :c:expr:`char * const *` + in the *keywords* parameter declaration of + :c:func:`PyArg_ParseTupleAndKeywords` and + :c:func:`PyArg_VaParseTupleAndKeywords`. + Default empty for C and ``const`` for C++ + (:c:expr:`const char * const *`). + To override, define it to the desired value before including + :file:`Python.h`. + + .. versionadded:: 3.13 + --------------- Building values @@ -547,7 +585,7 @@ Building values Same as ``s#``. ``u`` (:class:`str`) [const wchar_t \*] - Convert a null-terminated :c:expr:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) + Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) data to a Python Unicode object. If the Unicode buffer pointer is ``NULL``, ``None`` is returned. @@ -613,8 +651,10 @@ Building values Convert a C :c:type:`Py_complex` structure to a Python complex number. ``O`` (object) [PyObject \*] - Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed + Pass a Python object untouched but create a new + :term:`strong reference` to it + (i.e. its reference count is incremented by one). + If the object passed in is a ``NULL`` pointer, it is assumed that this was caused because the call producing the argument found an error and set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't raise an exception. If no exception has been raised yet, :exc:`SystemError` is @@ -624,7 +664,7 @@ Building values Same as ``O``. ``N`` (object) [PyObject \*] - Same as ``O``, except it doesn't increment the reference count on the object. + Same as ``O``, except it doesn't create a new :term:`strong reference`. Useful when the object is created by a call to an object constructor in the argument list. diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index c197d447e9618c..b4dc4849d044e1 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -6,11 +6,17 @@ Boolean Objects --------------- Booleans in Python are implemented as a subclass of integers. There are only -two booleans, :const:`Py_False` and :const:`Py_True`. As such, the normal +two booleans, :c:data:`Py_False` and :c:data:`Py_True`. As such, the normal creation and deletion functions don't apply to booleans. The following macros are available, however. +.. c:var:: PyTypeObject PyBool_Type + + This instance of :c:type:`PyTypeObject` represents the Python boolean type; it + is the same object as :class:`bool` in the Python layer. + + .. c:function:: int PyBool_Check(PyObject *o) Return true if *o* is of type :c:data:`PyBool_Type`. This function always @@ -19,29 +25,32 @@ are available, however. .. c:var:: PyObject* Py_False - The Python ``False`` object. This object has no methods. It needs to be - treated just like any other object with respect to reference counts. + The Python ``False`` object. This object has no methods and is + :term:`immortal`. + + .. versionchanged:: 3.12 + :c:data:`Py_False` is :term:`immortal`. .. c:var:: PyObject* Py_True - The Python ``True`` object. This object has no methods. It needs to be treated - just like any other object with respect to reference counts. + The Python ``True`` object. This object has no methods and is + :term:`immortal`. + + .. versionchanged:: 3.12 + :c:data:`Py_True` is :term:`immortal`. .. c:macro:: Py_RETURN_FALSE - Return :const:`Py_False` from a function, properly incrementing its reference - count. + Return :c:data:`Py_False` from a function. .. c:macro:: Py_RETURN_TRUE - Return :const:`Py_True` from a function, properly incrementing its reference - count. + Return :c:data:`Py_True` from a function. .. c:function:: PyObject* PyBool_FromLong(long v) - Return a new reference to :const:`Py_True` or :const:`Py_False` depending on the - truth value of *v*. + Return :c:data:`Py_True` or :c:data:`Py_False`, depending on the truth value of *v*. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 91d1edd9b2ec46..e572815ffd6259 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -44,7 +44,7 @@ the elements exposed by an :class:`array.array` can be multi-byte values. An example consumer of the buffer interface is the :meth:`~io.BufferedIOBase.write` method of file objects: any object that can export a series of bytes through -the buffer interface can be written to a file. While :meth:`write` only +the buffer interface can be written to a file. While :meth:`!write` only needs read-only access to the internal contents of the object passed to it, other methods such as :meth:`~io.BufferedIOBase.readinto` need write access to the contents of their argument. The buffer interface allows objects to @@ -102,7 +102,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:member:: PyObject *obj A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to ``NULL`` by + the consumer and automatically released + (i.e. reference count decremented) + and set to ``NULL`` by :c:func:`PyBuffer_Release`. The field is the equivalent of the return value of any standard C-API function. @@ -159,10 +161,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. - - The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions - to 64. Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. + The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. .. c:member:: Py_ssize_t *shape @@ -215,6 +214,17 @@ a buffer, see :c:func:`PyObject_GetBuffer`. freed when the buffer is released. The consumer MUST NOT alter this value. + +Constants: + +.. c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. + + .. _buffer-request-types: Buffer request types @@ -225,7 +235,7 @@ object via :c:func:`PyObject_GetBuffer`. Since the complexity of the logical structure of the memory can vary drastically, the consumer uses the *flags* argument to specify the exact buffer type it can handle. -All :c:data:`Py_buffer` fields are unambiguously defined by the request +All :c:type:`Py_buffer` fields are unambiguously defined by the request type. request-independent fields @@ -438,7 +448,7 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and + :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. On success, fill in *view*, set ``view->obj`` to a new reference @@ -454,7 +464,8 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) - Release the buffer *view* and decrement the reference count for + Release the buffer *view* and release the :term:`strong reference` + (i.e. decrement the reference count) to the view's supporting object, ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. @@ -464,7 +475,7 @@ Buffer-related functions .. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *format) - Return the implied :c:data:`~Py_buffer.itemsize` from :c:data:`~Py_buffer.format`. + Return the implied :c:member:`~Py_buffer.itemsize` from :c:member:`~Py_buffer.format`. On error, raise an exception and return -1. .. versionadded:: 3.9 @@ -524,7 +535,7 @@ Buffer-related functions and :c:macro:`PyBUF_WRITABLE` is set in *flags*. On success, set ``view->obj`` to a new reference to *exporter* and - return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set + return 0. Otherwise, raise :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc `, diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 9f48f2ffafe170..4790d3b2da4375 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -64,39 +64,39 @@ called with a non-bytes parameter. +-------------------+---------------+--------------------------------+ | Format Characters | Type | Comment | +===================+===============+================================+ - | :attr:`%%` | *n/a* | The literal % character. | + | ``%%`` | *n/a* | The literal % character. | +-------------------+---------------+--------------------------------+ - | :attr:`%c` | int | A single byte, | + | ``%c`` | int | A single byte, | | | | represented as a C int. | +-------------------+---------------+--------------------------------+ - | :attr:`%d` | int | Equivalent to | + | ``%d`` | int | Equivalent to | | | | ``printf("%d")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%u` | unsigned int | Equivalent to | + | ``%u`` | unsigned int | Equivalent to | | | | ``printf("%u")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%ld` | long | Equivalent to | + | ``%ld`` | long | Equivalent to | | | | ``printf("%ld")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%lu` | unsigned long | Equivalent to | + | ``%lu`` | unsigned long | Equivalent to | | | | ``printf("%lu")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%zd` | :c:type:`\ | Equivalent to | + | ``%zd`` | :c:type:`\ | Equivalent to | | | Py_ssize_t` | ``printf("%zd")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%zu` | size_t | Equivalent to | + | ``%zu`` | size_t | Equivalent to | | | | ``printf("%zu")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%i` | int | Equivalent to | + | ``%i`` | int | Equivalent to | | | | ``printf("%i")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%x` | int | Equivalent to | + | ``%x`` | int | Equivalent to | | | | ``printf("%x")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%s` | const char\* | A null-terminated C character | + | ``%s`` | const char\* | A null-terminated C character | | | | array. | +-------------------+---------------+--------------------------------+ - | :attr:`%p` | const void\* | The hex representation of a C | + | ``%p`` | const void\* | The hex representation of a C | | | | pointer. Mostly equivalent to | | | | ``printf("%p")`` except that | | | | it is guaranteed to start with | @@ -155,6 +155,7 @@ called with a non-bytes parameter. Return the null-terminated contents of the object *obj* through the output variables *buffer* and *length*. + Returns ``0`` on success. If *length* is ``NULL``, the bytes object may not contain embedded null bytes; @@ -184,8 +185,8 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*. This version decrements the reference count of - *newpart*. + appended to *bytes*. This version releases the :term:`strong reference` + to *newpart* (i.e. decrements its reference count). .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 4dc66e318cd12e..7198d6bc056eb4 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -59,12 +59,12 @@ This bears repeating: .. versionchanged:: 3.12 - The :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag is now removed from a class + The :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag is now removed from a class when the class's :py:meth:`~object.__call__` method is reassigned. (This internally sets :c:member:`~PyTypeObject.tp_call` only, and thus may make it behave differently than the vectorcall function.) In earlier Python versions, vectorcall should only be used with - :const:`immutable ` or static types. + :c:macro:`immutable ` or static types. A class should not implement vectorcall if that would be slower than *tp_call*. For example, if the callee needs to convert @@ -72,7 +72,7 @@ the arguments to an args tuple and kwargs dict anyway, then there is no point in implementing vectorcall. Classes can implement the vectorcall protocol by enabling the -:const:`Py_TPFLAGS_HAVE_VECTORCALL` flag and setting +:c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag and setting :c:member:`~PyTypeObject.tp_vectorcall_offset` to the offset inside the object structure where a *vectorcallfunc* appears. This is a pointer to a function with the following signature: @@ -84,7 +84,7 @@ This is a pointer to a function with the following signature: values of the keyword arguments. This can be *NULL* if there are no arguments. - *nargsf* is the number of positional arguments plus possibly the - :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` flag. + :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET` flag. To get the actual number of positional arguments from *nargsf*, use :c:func:`PyVectorcall_NARGS`. - *kwnames* is a tuple containing the names of the keyword arguments; @@ -93,7 +93,7 @@ This is a pointer to a function with the following signature: and they must be unique. If there are no keyword arguments, then *kwnames* can instead be *NULL*. -.. data:: PY_VECTORCALL_ARGUMENTS_OFFSET +.. c:macro:: PY_VECTORCALL_ARGUMENTS_OFFSET If this flag is set in a vectorcall *nargsf* argument, the callee is allowed to temporarily change ``args[-1]``. In other words, *args* points to @@ -104,28 +104,17 @@ This is a pointer to a function with the following signature: ``args[0]`` may be changed. Whenever they can do so cheaply (without additional allocation), callers - are encouraged to use :const:`PY_VECTORCALL_ARGUMENTS_OFFSET`. + are encouraged to use :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET`. Doing so will allow callables such as bound methods to make their onward calls (which include a prepended *self* argument) very efficiently. + .. versionadded:: 3.8 + To call an object that implements vectorcall, use a :ref:`call API ` function as with any other callable. :c:func:`PyObject_Vectorcall` will usually be most efficient. -.. note:: - - In CPython 3.8, the vectorcall API and related functions were available - provisionally under names with a leading underscore: - ``_PyObject_Vectorcall``, ``_Py_TPFLAGS_HAVE_VECTORCALL``, - ``_PyObject_VectorcallMethod``, ``_PyVectorcall_Function``, - ``_PyObject_CallOneArg``, ``_PyObject_CallMethodNoArgs``, - ``_PyObject_CallMethodOneArg``. - Additionally, ``PyObject_VectorcallDict`` was available as - ``_PyObject_FastCallDict``. - The old names are still defined as aliases of the new, non-underscored names. - - Recursion Control ................. @@ -165,7 +154,7 @@ Vectorcall Support API This is mostly useful to check whether or not *op* supports vectorcall, which can be done by checking ``PyVectorcall_Function(op) != NULL``. - .. versionadded:: 3.8 + .. versionadded:: 3.9 .. c:function:: PyObject* PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict) @@ -174,7 +163,7 @@ Vectorcall Support API This is a specialized function, intended to be put in the :c:member:`~PyTypeObject.tp_call` slot or be used in an implementation of ``tp_call``. - It does not check the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag + It does not check the :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag and it does not fall back to ``tp_call``. .. versionadded:: 3.8 @@ -392,11 +381,11 @@ please see individual documentation for details. *args[0]*, and the *args* array starting at *args[1]* represents the arguments of the call. There must be at least one positional argument. *nargsf* is the number of positional arguments including *args[0]*, - plus :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` if the value of ``args[0]`` may + plus :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET` if the value of ``args[0]`` may temporarily be changed. Keyword arguments can be passed just like in :c:func:`PyObject_Vectorcall`. - If the object has the :const:`Py_TPFLAGS_METHOD_DESCRIPTOR` feature, + If the object has the :c:macro:`Py_TPFLAGS_METHOD_DESCRIPTOR` feature, this will call the unbound method object with the full *args* vector as arguments. diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 427ed959c58568..cdb8aa33e9fd32 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -64,7 +64,7 @@ Refer to :ref:`using-capsules` for more information on using these objects. The *name* parameter must compare exactly to the name stored in the capsule. If the name stored in the capsule is ``NULL``, the *name* passed in must also - be ``NULL``. Python uses the C function :c:func:`strcmp` to compare capsule + be ``NULL``. Python uses the C function :c:func:`!strcmp` to compare capsule names. @@ -121,7 +121,7 @@ Refer to :ref:`using-capsules` for more information on using these objects. compared.) In other words, if :c:func:`PyCapsule_IsValid` returns a true value, calls to - any of the accessors (any function starting with :c:func:`PyCapsule_Get`) are + any of the accessors (any function starting with ``PyCapsule_Get``) are guaranteed to succeed. Return a nonzero value if the object is valid and matches the name passed in. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index ac4ef5adc5cc20..f8cd0344fdd1c0 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -25,7 +25,7 @@ Cell objects are not likely to be useful elsewhere. The type object corresponding to cell objects. -.. c:function:: int PyCell_Check(ob) +.. c:function:: int PyCell_Check(PyObject *ob) Return true if *ob* is a cell object; *ob* must not be ``NULL``. This function always succeeds. diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index a99de9904c0740..5082b0cb6ad3f3 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -33,20 +33,20 @@ bound into a function. Return the number of free variables in *co*. -.. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable) +.. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Return a new code object. If you need a dummy code object to create a frame, use :c:func:`PyCode_NewEmpty` instead. Since the definition of the bytecode changes often, calling - :c:func:`PyCode_New` directly can bind you to a precise Python version. + :c:func:`PyUnstable_Code_New` directly can bind you to a precise Python version. The many arguments of this function are inter-dependent in complex ways, meaning that subtle changes to values are likely to result in incorrect execution or VM crashes. Use this function only with extreme care. .. versionchanged:: 3.11 - Added ``exceptiontable`` parameter. + Added ``qualname`` and ``exceptiontable`` parameters. .. index:: single: PyCode_New @@ -56,17 +56,17 @@ bound into a function. The old name is deprecated, but will remain available until the signature changes again. -.. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable) +.. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) - Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments. - The same caveats that apply to ``PyCode_New`` also apply to this function. + Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. + The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. .. index:: single: PyCode_NewWithPosOnlyArgs .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` .. versionchanged:: 3.11 - Added ``exceptiontable`` parameter. + Added ``qualname`` and ``exceptiontable`` parameters. .. versionchanged:: 3.12 diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 235c77c945cc5b..8ae5c4fecd6248 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -7,7 +7,7 @@ Codec registry and support functions Register a new codec search function. - As side effect, this tries to load the :mod:`encodings` package, if not yet + As side effect, this tries to load the :mod:`!encodings` package, if not yet done, to make sure that it is always first in the list of search functions. .. c:function:: int PyCodec_Unregister(PyObject *search_function) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 344da903da4c1a..e3fd001c599c80 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -64,7 +64,7 @@ pointers. This is consistent throughout the API. representation. If *divisor* is null, this method returns zero and sets - :c:data:`errno` to :c:data:`EDOM`. + :c:data:`errno` to :c:macro:`!EDOM`. .. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) @@ -73,7 +73,7 @@ pointers. This is consistent throughout the API. representation. If *num* is null and *exp* is not a positive real number, - this method returns zero and sets :c:data:`errno` to :c:data:`EDOM`. + this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. Complex Numbers as Python Objects @@ -127,12 +127,12 @@ Complex Numbers as Python Objects Return the :c:type:`Py_complex` value of the complex number *op*. - If *op* is not a Python complex number object but has a :meth:`__complex__` + If *op* is not a Python complex number object but has a :meth:`~object.__complex__` method, this method will first be called to convert *op* to a Python complex - number object. If ``__complex__()`` is not defined then it falls back to - :meth:`__float__`. If ``__float__()`` is not defined then it falls back - to :meth:`__index__`. Upon failure, this method returns ``-1.0`` as a real + number object. If :meth:`!__complex__` is not defined then it falls back to + :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + to :meth:`~object.__index__`. Upon failure, this method returns ``-1.0`` as a real value. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index fdb321fe7ab3f2..c5350123dfdfdc 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -119,10 +119,10 @@ The following functions provide locale-independent string to number conversions. .. c:function:: int PyOS_stricmp(const char *s1, const char *s2) Case insensitive comparison of strings. The function works almost - identically to :c:func:`strcmp` except that it ignores the case. + identically to :c:func:`!strcmp` except that it ignores the case. .. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) Case insensitive comparison of strings. The function works almost - identically to :c:func:`strncmp` except that it ignores the case. + identically to :c:func:`!strncmp` except that it ignores the case. diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 72fc07afbf1f4d..97522da773477e 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,54 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following macros. +.. c:type:: PyDateTime_Date + + This subtype of :c:type:`PyObject` represents a Python date object. + +.. c:type:: PyDateTime_DateTime + + This subtype of :c:type:`PyObject` represents a Python datetime object. + +.. c:type:: PyDateTime_Time + + This subtype of :c:type:`PyObject` represents a Python time object. + +.. c:type:: PyDateTime_Delta + + This subtype of :c:type:`PyObject` represents the difference between two datetime values. + +.. c:var:: PyTypeObject PyDateTime_DateType + + This instance of :c:type:`PyTypeObject` represents the Python date type; + it is the same object as :class:`datetime.date` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DateTimeType + + This instance of :c:type:`PyTypeObject` represents the Python datetime type; + it is the same object as :class:`datetime.datetime` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TimeType + + This instance of :c:type:`PyTypeObject` represents the Python time type; + it is the same object as :class:`datetime.time` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DeltaType + + This instance of :c:type:`PyTypeObject` represents Python type for + the difference between two datetime values; + it is the same object as :class:`datetime.timedelta` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TZInfoType + + This instance of :c:type:`PyTypeObject` represents the Python time zone info type; + it is the same object as :class:`datetime.tzinfo` in the Python layer. + + Macro for access to the UTC singleton: .. c:var:: PyObject* PyDateTime_TimeZone_UTC @@ -28,7 +71,7 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateType`. *ob* must not be ``NULL``. This function always succeeds. @@ -41,7 +84,7 @@ Type-check macros: .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -54,7 +97,7 @@ Type-check macros: .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -67,7 +110,7 @@ Type-check macros: .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always succeeds. @@ -80,7 +123,7 @@ Type-check macros: .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always succeeds. @@ -133,7 +176,7 @@ Macros to create objects: :class:`datetime.timedelta` objects. -.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset) +.. c:function:: PyObject* PyTimeZone_FromOffset(PyObject *offset) Return a :class:`datetime.timezone` object with an unnamed fixed offset represented by the *offset* argument. @@ -141,7 +184,7 @@ Macros to create objects: .. versionadded:: 3.7 -.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name) +.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyObject *offset, PyObject *name) Return a :class:`datetime.timezone` object with a fixed offset represented by the *offset* argument and with tzname *name*. @@ -150,8 +193,8 @@ Macros to create objects: Macros to extract fields from date objects. The argument must be an instance of -:c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is +:c:type:`PyDateTime_Date`, including subclasses (such as +:c:type:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) @@ -170,7 +213,7 @@ not checked: Macros to extract fields from datetime objects. The argument must be an -instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument +instance of :c:type:`PyDateTime_DateTime`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) @@ -208,7 +251,7 @@ must not be ``NULL``, and the type is not checked: Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, +:c:type:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) @@ -246,7 +289,7 @@ and the type is not checked: Macros to extract fields from time delta objects. The argument must be an -instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must +instance of :c:type:`PyDateTime_Delta`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 0ca8ad624b2034..8471c98d044872 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -55,6 +55,15 @@ Dictionary Objects This is equivalent to the Python expression ``key in p``. +.. c:function:: int PyDict_ContainsString(PyObject *p, const char *key) + + This is the same as :c:func:`PyDict_Contains`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: PyObject* PyDict_Copy(PyObject *p) Return a new dictionary that contains the same key-value pairs as *p*. @@ -70,12 +79,9 @@ Dictionary Objects .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) - .. index:: single: PyUnicode_FromString() - - Insert *val* into the dictionary *p* using *key* as a key. *key* should - be a :c:expr:`const char*`. The key object is created using - ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on - failure. This function *does not* steal a reference to *val*. + This is the same as :c:func:`PyDict_SetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) @@ -88,19 +94,37 @@ Dictionary Objects .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) - Remove the entry in dictionary *p* which has a key specified by the string *key*. - If *key* is not in the dictionary, :exc:`KeyError` is raised. - Return ``0`` on success or ``-1`` on failure. + This is the same as :c:func:`PyDict_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + +.. c:function:: int PyDict_GetItemRef(PyObject *p, PyObject *key, PyObject **result) + + Return a new :term:`strong reference` to the object from dictionary *p* + which has a key *key*: + + * If the key is present, set *\*result* to a new :term:`strong reference` + to the value and return ``1``. + * If the key is missing, set *\*result* to ``NULL`` and return ``0``. + * On error, raise an exception and return ``-1``. + + .. versionadded:: 3.13 + + See also the :c:func:`PyObject_GetItem` function. .. c:function:: PyObject* PyDict_GetItem(PyObject *p, PyObject *key) - Return the object from dictionary *p* which has a key *key*. Return ``NULL`` - if the key *key* is not present, but *without* setting an exception. + Return a :term:`borrowed reference` to the object from dictionary *p* which + has a key *key*. Return ``NULL`` if the key *key* is missing *without* + setting an exception. + + .. note:: - Note that exceptions which occur while calling :meth:`__hash__` and - :meth:`__eq__` methods will get suppressed. - To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. + Exceptions that occur while this calls :meth:`~object.__hash__` and + :meth:`~object.__eq__` methods are silently ignored. + Prefer the :c:func:`PyDict_GetItemWithError` function instead. .. versionchanged:: 3.10 Calling this API without :term:`GIL` held had been allowed for historical @@ -118,12 +142,25 @@ Dictionary Objects .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:expr:`const char*`, rather than a :c:expr:`PyObject*`. + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. note:: - Note that exceptions which occur while calling :meth:`__hash__` and - :meth:`__eq__` methods and creating a temporary string object - will get suppressed. - To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. + Exceptions that occur while this calls :meth:`~object.__hash__` and + :meth:`~object.__eq__` methods or while creating the temporary :class:`str` + object are silently ignored. + Prefer using the :c:func:`PyDict_GetItemWithError` function with your own + :c:func:`PyUnicode_FromString` *key* instead. + + +.. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) + + Similar than :c:func:`PyDict_GetItemRef`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. versionadded:: 3.13 .. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj) @@ -136,6 +173,33 @@ Dictionary Objects .. versionadded:: 3.4 + +.. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) + + Remove *key* from dictionary *p* and optionally return the removed value. + Do not raise :exc:`KeyError` if the key missing. + + - If the key is present, set *\*result* to a new reference to the removed + value if *result* is not ``NULL``, and return ``1``. + - If the key is missing, set *\*result* to ``NULL`` if *result* is not + ``NULL``, and return ``0``. + - On error, raise an exception and return ``-1``. + + This is similar to :meth:`dict.pop`, but without the default value and + not raising :exc:`KeyError` if the key missing. + + .. versionadded:: 3.13 + + +.. c:function:: int PyDict_PopString(PyObject *p, const char *key, PyObject **result) + + Similar to :c:func:`PyDict_Pop`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: PyObject* PyDict_Items(PyObject *p) Return a :c:type:`PyListObject` containing all the items from the dictionary. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 4ed96f01dbbc3e..a3a63b38c432f2 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -60,9 +60,14 @@ Printing and clearing Call this function **only** when the error indicator is set. Otherwise it will cause a fatal error! - If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, - :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the - type, value and traceback of the printed exception, respectively. + If *set_sys_last_vars* is nonzero, the variable :data:`sys.last_exc` is + set to the printed exception. For backwards compatibility, the + deprecated variables :data:`sys.last_type`, :data:`sys.last_value` and + :data:`sys.last_traceback` are also set to the type, value and traceback + of this exception, respectively. + + .. versionchanged:: 3.12 + The setting of :data:`sys.last_exc` was added. .. c:function:: void PyErr_Print() @@ -78,14 +83,34 @@ Printing and clearing This utility function prints a warning message to ``sys.stderr`` when an exception has been set but it is impossible for the interpreter to actually raise the exception. It is used, for example, when an exception occurs in an - :meth:`__del__` method. + :meth:`~object.__del__` method. The function is called with a single argument *obj* that identifies the context in which the unraisable exception occurred. If possible, the repr of *obj* will be printed in the warning message. + If *obj* is ``NULL``, only the traceback is printed. An exception must be set when calling this function. + .. versionchanged:: 3.4 + Print a traceback. Print only traceback if *obj* is ``NULL``. + + .. versionchanged:: 3.8 + Use :func:`sys.unraisablehook`. + + +.. c:function:: void PyErr_FormatUnraisable(const char *format, ...) + + Similar to :c:func:`PyErr_WriteUnraisable`, but the *format* and subsequent + parameters help format the warning message; they have the same meaning and + values as in :c:func:`PyUnicode_FromFormat`. + ``PyErr_WriteUnraisable(obj)`` is roughtly equivalent to + ``PyErr_FormatUnraisable("Exception ignored in: %R, obj)``. + If *format* is ``NULL``, only the traceback is printed. + + .. versionadded:: 3.13 + + .. c:function:: void PyErr_DisplayException(PyObject *exc) Print the standard traceback display of ``exc`` to ``sys.stderr``, including @@ -93,6 +118,7 @@ Printing and clearing .. versionadded:: 3.12 + Raising exceptions ================== @@ -105,7 +131,8 @@ For convenience, some of these functions will always return a This is the most common way to set the error indicator. The first argument specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + e.g. :c:data:`PyExc_RuntimeError`. You need not create a new + :term:`strong reference` to it (e.g. with :c:func:`Py_INCREF`). The second argument is an error message; it is decoded from ``'utf-8'``. @@ -158,9 +185,9 @@ For convenience, some of these functions will always return a This is a convenience function to raise an exception when a C library function has returned an error and set the C variable :c:data:`errno`. It constructs a tuple object whose first item is the integer :c:data:`errno` value and whose - second item is the corresponding error message (gotten from :c:func:`strerror`), + second item is the corresponding error message (gotten from :c:func:`!strerror`), and then calls ``PyErr_SetObject(type, object)``. On Unix, when the - :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, + :c:data:`errno` value is :c:macro:`!EINTR`, indicating an interrupted system call, this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, leaves it set to that. The function always returns ``NULL``, so a wrapper function around a system call can write ``return PyErr_SetFromErrno(type);`` @@ -172,7 +199,7 @@ For convenience, some of these functions will always return a Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as a third parameter. In the case of :exc:`OSError` exception, - this is used to define the :attr:`filename` attribute of the + this is used to define the :attr:`!filename` attribute of the exception instance. @@ -195,12 +222,12 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) This is a convenience function to raise :exc:`WindowsError`. If called with - *ierr* of ``0``, the error code returned by a call to :c:func:`GetLastError` - is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve - the Windows description of error code given by *ierr* or :c:func:`GetLastError`, + *ierr* of ``0``, the error code returned by a call to :c:func:`!GetLastError` + is used instead. It calls the Win32 function :c:func:`!FormatMessage` to retrieve + the Windows description of error code given by *ierr* or :c:func:`!GetLastError`, then it constructs a tuple object whose first item is the *ierr* value and whose second item is the corresponding error message (gotten from - :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, object)``. This function always returns ``NULL``. .. availability:: Windows. @@ -216,17 +243,21 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). + Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is decoded from the filesystem + encoding (:func:`os.fsdecode`) and passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. .. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. + Similar to :c:func:`PyErr_SetExcFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. @@ -626,7 +657,7 @@ Signal Handling be interruptible by user requests (such as by pressing Ctrl-C). .. note:: - The default Python signal handler for :const:`SIGINT` raises the + The default Python signal handler for :c:macro:`!SIGINT` raises the :exc:`KeyboardInterrupt` exception. @@ -637,7 +668,7 @@ Signal Handling single: SIGINT single: KeyboardInterrupt (built-in exception) - Simulate the effect of a :const:`SIGINT` signal arriving. + Simulate the effect of a :c:macro:`!SIGINT` signal arriving. This is equivalent to ``PyErr_SetInterruptEx(SIGINT)``. .. note:: @@ -661,7 +692,7 @@ Signal Handling to interrupt an operation). If the given signal isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), it will be ignored. + :py:const:`signal.SIG_DFL` or :py:const:`signal.SIG_IGN`), it will be ignored. If *signum* is outside of the allowed range of signal numbers, ``-1`` is returned. Otherwise, ``0`` is returned. The error indicator is @@ -749,7 +780,7 @@ Exception Objects .. c:function:: PyObject* PyException_GetCause(PyObject *ex) - Return the cause (either an exception instance, or :const:`None`, + Return the cause (either an exception instance, or ``None``, set by ``raise ... from ...``) associated with the exception as a new reference, as accessible from Python through :attr:`__cause__`. @@ -758,7 +789,7 @@ Exception Objects Set the cause associated with the exception to *cause*. Use ``NULL`` to clear it. There is no type check to make sure that *cause* is either an exception - instance or :const:`None`. This steals a reference to *cause*. + instance or ``None``. This steals a reference to *cause*. :attr:`__suppress_context__` is implicitly set to ``True`` by this function. @@ -772,6 +803,18 @@ Exception Objects Set :attr:`~BaseException.args` of exception *ex* to *args*. +.. c:function:: PyObject* PyUnstable_Exc_PrepReraiseStar(PyObject *orig, PyObject *excs) + + Implement part of the interpreter's implementation of :keyword:`!except*`. + *orig* is the original exception that was caught, and *excs* is the list of + the exceptions that need to be raised. This list contains the unhandled + part of *orig*, if any, as well as the exceptions that were raised from the + :keyword:`!except*` clauses (so they have a different traceback from *orig*) and + those that were reraised (and have the same traceback as *orig*). + Return the :exc:`ExceptionGroup` that needs to be reraised in the end, or + ``None`` if there is nothing to reraise. + + .. versionadded:: 3.12 .. _unicodeexceptions: @@ -857,7 +900,7 @@ because the :ref:`call protocol ` takes care of recursion handling. Marks a point where a recursive C-level call is about to be performed. - If :const:`USE_STACKCHECK` is defined, this function checks if the OS + If :c:macro:`USE_STACKCHECK` is defined, this function checks if the OS stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it sets a :exc:`MemoryError` and returns a nonzero value. @@ -870,7 +913,7 @@ because the :ref:`call protocol ` takes care of recursion handling. depth limit. .. versionchanged:: 3.9 - This function is now also available in the limited API. + This function is now also available in the :ref:`limited API `. .. c:function:: void Py_LeaveRecursiveCall(void) @@ -878,7 +921,7 @@ because the :ref:`call protocol ` takes care of recursion handling. *successful* invocation of :c:func:`Py_EnterRecursiveCall`. .. versionchanged:: 3.9 - This function is now also available in the limited API. + This function is now also available in the :ref:`limited API `. Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires special recursion handling. In addition to protecting the stack, diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index f32ecba9f27029..b36c800e00444a 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -93,7 +93,7 @@ the :mod:`io` APIs instead. .. index:: single: Py_PRINT_RAW Write object *obj* to file object *p*. The only supported flag for *flags* is - :const:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written + :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the appropriate exception will be set. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 05b2d100d575cb..4f6ac0d8175c6b 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -3,7 +3,7 @@ .. _floatobjects: Floating Point Objects ----------------------- +====================== .. index:: pair: object; floating point @@ -45,14 +45,14 @@ Floating Point Objects .. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) Return a C :c:expr:`double` representation of the contents of *pyfloat*. If - *pyfloat* is not a Python floating point object but has a :meth:`__float__` + *pyfloat* is not a Python floating point object but has a :meth:`~object.__float__` method, this method will first be called to convert *pyfloat* into a float. - If ``__float__()`` is not defined then it falls back to :meth:`__index__`. + If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. This method returns ``-1.0`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) @@ -79,7 +79,7 @@ Floating Point Objects Pack and Unpack functions -========================= +------------------------- The pack and unpack functions provide an efficient platform-independent way to store floating-point values as byte strings. The Pack routines produce a bytes @@ -104,12 +104,12 @@ happens in such cases is partly accidental (alas). .. versionadded:: 3.11 Pack functions --------------- +^^^^^^^^^^^^^^ The pack routines write 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if you want the bytes string in little-endian format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` ``p+7``), zero if you -want big-endian format (exponent first, at *p*). The :c:data:`PY_BIG_ENDIAN` +want big-endian format (exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` constant can be used to use the native endian: it is equal to ``1`` on big endian processor, or ``0`` on little endian processor. @@ -135,12 +135,12 @@ There are two problems on non-IEEE platforms: Unpack functions ----------------- +^^^^^^^^^^^^^^^^ The unpack routines read 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if the bytes string is in little-endian format (exponent last, at ``p+1``, ``p+3`` or ``p+6`` and ``p+7``), zero if big-endian -(exponent first, at *p*). The :c:data:`PY_BIG_ENDIAN` constant can be used to +(exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` constant can be used to use the native endian: it is equal to ``1`` on big endian processor, or ``0`` on little endian processor. diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1ac8f03d6e48f8..6bb1e9b5803b58 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -50,7 +50,7 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) - Get the *frame*'s ``f_builtins`` attribute. + Get the *frame*'s :attr:`~frame.f_builtins` attribute. Return a :term:`strong reference`. The result cannot be ``NULL``. @@ -81,7 +81,7 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) - Get the *frame*'s ``f_globals`` attribute. + Get the *frame*'s :attr:`~frame.f_globals` attribute. Return a :term:`strong reference`. The result cannot be ``NULL``. @@ -90,7 +90,7 @@ See also :ref:`Reflection `. .. c:function:: int PyFrame_GetLasti(PyFrameObject *frame) - Get the *frame*'s ``f_lasti`` attribute. + Get the *frame*'s :attr:`~frame.f_lasti` attribute. Returns -1 if ``frame.f_lasti`` is ``None``. @@ -120,7 +120,7 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) - Get the *frame*'s ``f_locals`` attribute (:class:`dict`). + Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`). Return a :term:`strong reference`. @@ -130,3 +130,38 @@ See also :ref:`Reflection `. .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) Return the line number that *frame* is currently executing. + + + +Internal Frames +^^^^^^^^^^^^^^^ + +Unless using :pep:`523`, you will not need this. + +.. c:struct:: _PyInterpreterFrame + + The interpreter's internal frame representation. + + .. versionadded:: 3.11 + +.. c:function:: PyObject* PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); + + Return a :term:`strong reference` to the code object for the frame. + + .. versionadded:: 3.12 + + +.. c:function:: int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); + + Return the byte offset into the last executed instruction. + + .. versionadded:: 3.12 + + +.. c:function:: int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); + + Return the currently executing line number, or -1 if there is no line number. + + .. versionadded:: 3.12 + + diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 5857dba82c11c6..0a18e63c7e7a2c 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -37,7 +37,7 @@ There are a few functions specific to Python functions. The function's docstring and name are retrieved from the code object. *__module__* is retrieved from *globals*. The argument defaults, annotations and closure are set to ``NULL``. *__qualname__* is set to the same value as the code object's - ``co_qualname`` field. + :attr:`~codeobject.co_qualname` field. .. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) @@ -45,7 +45,7 @@ There are a few functions specific to Python functions. As :c:func:`PyFunction_New`, but also allows setting the function object's ``__qualname__`` attribute. *qualname* should be a unicode object or ``NULL``; if ``NULL``, the ``__qualname__`` attribute is set to the same value as the - code object's ``co_qualname`` field. + code object's :attr:`~codeobject.co_qualname` field. .. versionadded:: 3.3 diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index c3260a21bc7f8b..6b2494ee4f0ed4 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -13,22 +13,20 @@ or strings), do not need to provide any explicit support for garbage collection. To create a container type, the :c:member:`~PyTypeObject.tp_flags` field of the type object must -include the :const:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the +include the :c:macro:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the :c:member:`~PyTypeObject.tp_traverse` handler. If instances of the type are mutable, a :c:member:`~PyTypeObject.tp_clear` implementation must also be provided. -.. data:: Py_TPFLAGS_HAVE_GC - :noindex: - +:c:macro:`Py_TPFLAGS_HAVE_GC` Objects with a type with this flag set must conform with the rules documented here. For convenience these objects will be referred to as container objects. Constructors for container types must conform to two rules: -#. The memory for the object must be allocated using :c:func:`PyObject_GC_New` - or :c:func:`PyObject_GC_NewVar`. +#. The memory for the object must be allocated using :c:macro:`PyObject_GC_New` + or :c:macro:`PyObject_GC_NewVar`. #. Once all the fields which may contain references to other containers are initialized, it must call :c:func:`PyObject_GC_Track`. @@ -52,21 +50,21 @@ rules: :c:member:`~PyTypeObject.tp_flags`, :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields if the type inherits from a class that implements the garbage collector protocol and the child class - does *not* include the :const:`Py_TPFLAGS_HAVE_GC` flag. + does *not* include the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -.. c:function:: TYPE* PyObject_GC_New(TYPE, PyTypeObject *type) +.. c:macro:: PyObject_GC_New(TYPE, typeobj) - Analogous to :c:func:`PyObject_New` but for container objects with the - :const:`Py_TPFLAGS_HAVE_GC` flag set. + Analogous to :c:macro:`PyObject_New` but for container objects with the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. -.. c:function:: TYPE* PyObject_GC_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) +.. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size) - Analogous to :c:func:`PyObject_NewVar` but for container objects with the - :const:`Py_TPFLAGS_HAVE_GC` flag set. + Analogous to :c:macro:`PyObject_NewVar` but for container objects with the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. .. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) - Analogous to :c:func:`PyObject_GC_New` but allocates *extra_size* + Analogous to :c:macro:`PyObject_GC_New` but allocates *extra_size* bytes at the end of the object (at offset :c:member:`~PyTypeObject.tp_basicsize`). The allocated memory is initialized to zeros, @@ -87,7 +85,7 @@ rules: .. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) - Resize an object allocated by :c:func:`PyObject_NewVar`. Returns the + Resize an object allocated by :c:macro:`PyObject_NewVar`. Returns the resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet. @@ -130,8 +128,8 @@ rules: .. c:function:: void PyObject_GC_Del(void *op) - Releases memory allocated to an object using :c:func:`PyObject_GC_New` or - :c:func:`PyObject_GC_NewVar`. + Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or + :c:macro:`PyObject_GC_NewVar`. .. c:function:: void PyObject_GC_UnTrack(void *op) @@ -145,7 +143,7 @@ rules: .. versionchanged:: 3.8 - The :c:func:`_PyObject_GC_TRACK` and :c:func:`_PyObject_GC_UNTRACK` macros + The :c:func:`!_PyObject_GC_TRACK` and :c:func:`!_PyObject_GC_UNTRACK` macros have been removed from the public C API. The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type: diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst new file mode 100644 index 00000000000000..91d88ae27bc9f4 --- /dev/null +++ b/Doc/c-api/hash.rst @@ -0,0 +1,61 @@ +.. highlight:: c + +PyHash API +---------- + +See also the :c:member:`PyTypeObject.tp_hash` member. + +.. c:type:: Py_hash_t + + Hash value type: signed integer. + + .. versionadded:: 3.2 + +.. c:type:: Py_uhash_t + + Hash value type: unsigned integer. + + .. versionadded:: 3.2 + + +.. c:type:: PyHash_FuncDef + + Hash function definition used by :c:func:`PyHash_GetFuncDef`. + + .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + + Hash function. + + .. c:member:: const char *name + + Hash function name (UTF-8 encoded string). + + .. c:member:: const int hash_bits + + Internal size of the hash value in bits. + + .. c:member:: const int seed_bits + + Size of seed input in bits. + + .. versionadded:: 3.4 + + +.. c:function:: PyHash_FuncDef* PyHash_GetFuncDef(void) + + Get the hash function definition. + + .. seealso:: + :pep:`456` "Secure and interchangeable hash algorithm". + + .. versionadded:: 3.4 + + +.. c:function:: Py_hash_t Py_HashPointer(const void *ptr) + + Hash a pointer value: process the pointer value as an integer (cast it to + ``uintptr_t`` internally). The pointer is not dereferenced. + + The function cannot fail: it cannot return ``-1``. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 79843ba521ab93..51c20b202f091c 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -38,6 +38,9 @@ Importing Modules to per-module locks for most purposes, so this function's special behaviour isn't needed anymore. + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyImport_ImportModule` instead. + .. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) @@ -95,27 +98,40 @@ Importing Modules an exception set on failure (the module still exists in this case). -.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name) +.. c:function:: PyObject* PyImport_AddModuleRef(const char *name) + + Return the module object corresponding to a module name. + + The *name* argument may be of the form ``package.module``. First check the + modules dictionary if there's one there, and if not, create a new one and + insert it in the modules dictionary. - Return the module object corresponding to a module name. The *name* argument - may be of the form ``package.module``. First check the modules dictionary if - there's one there, and if not, create a new one and insert it in the modules - dictionary. Return ``NULL`` with an exception set on failure. + Return a :term:`strong reference` to the module on success. Return ``NULL`` + with an exception set on failure. - .. note:: + The module name *name* is decoded from UTF-8. - This function does not load or import the module; if the module wasn't already - loaded, you will get an empty module object. Use :c:func:`PyImport_ImportModule` - or one of its variants to import a module. Package structures implied by a - dotted name for *name* are not created if not already present. + This function does not load or import the module; if the module wasn't + already loaded, you will get an empty module object. Use + :c:func:`PyImport_ImportModule` or one of its variants to import a module. + Package structures implied by a dotted name for *name* are not created if + not already present. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name) + + Similar to :c:func:`PyImport_AddModuleRef`, but return a :term:`borrowed + reference` and *name* is a Python :class:`str` object. .. versionadded:: 3.3 .. c:function:: PyObject* PyImport_AddModule(const char *name) - Similar to :c:func:`PyImport_AddModuleObject`, but the name is a UTF-8 - encoded string instead of a Unicode object. + Similar to :c:func:`PyImport_AddModuleRef`, but return a :term:`borrowed + reference`. .. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co) @@ -126,19 +142,19 @@ Importing Modules read from a Python bytecode file or obtained from the built-in function :func:`compile`, load the module. Return a new reference to the module object, or ``NULL`` with an exception set if an error occurred. *name* - is removed from :attr:`sys.modules` in error cases, even if *name* was already - in :attr:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving - incompletely initialized modules in :attr:`sys.modules` is dangerous, as imports of + is removed from :data:`sys.modules` in error cases, even if *name* was already + in :data:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving + incompletely initialized modules in :data:`sys.modules` is dangerous, as imports of such modules have no way to know that the module object is an unknown (and probably damaged with respect to the module author's intents) state. The module's :attr:`__spec__` and :attr:`__loader__` will be set, if not set already, with the appropriate values. The spec's loader will be set to the module's ``__loader__`` (if set) and to an instance of - :class:`SourceFileLoader` otherwise. + :class:`~importlib.machinery.SourceFileLoader` otherwise. The module's :attr:`__file__` attribute will be set to the code object's - :c:member:`co_filename`. If applicable, :attr:`__cached__` will also + :attr:`~codeobject.co_filename`. If applicable, :attr:`__cached__` will also be set. This function will reload the module if it was already imported. See @@ -225,7 +241,7 @@ Importing Modules .. c:function:: PyObject* PyImport_GetImporter(PyObject *path) - Return a finder object for a :data:`sys.path`/:attr:`pkg.__path__` item + Return a finder object for a :data:`sys.path`/:attr:`!pkg.__path__` item *path*, possibly by fetching it from the :data:`sys.path_importer_cache` dict. If it wasn't yet cached, traverse :data:`sys.path_hooks` until a hook is found that can handle the path item. Return ``None`` if no hook could; @@ -294,23 +310,25 @@ Importing Modules .. c:struct:: _inittab - Structure describing a single entry in the list of built-in modules. Each of - these structures gives the name and initialization function for a module built - into the interpreter. The name is an ASCII encoded string. Programs which + Structure describing a single entry in the list of built-in modules. + Programs which embed Python may use an array of these structures in conjunction with :c:func:`PyImport_ExtendInittab` to provide additional built-in modules. - The structure is defined in :file:`Include/import.h` as:: + The structure consists of two members: - struct _inittab { - const char *name; /* ASCII encoded string */ - PyObject* (*initfunc)(void); - }; + .. c:member:: const char *name + + The module name, as an ASCII encoded string. + + .. c: member:: PyObject* (*initfunc)(void) + + Initialization function for a module built into the interpreter. .. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) Add a collection of modules to the table of built-in modules. The *newtab* - array must end with a sentinel entry which contains ``NULL`` for the :attr:`name` + array must end with a sentinel entry which contains ``NULL`` for the :c:member:`~_inittab.name` field; failure to provide the sentinel value can result in a memory fault. Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to extend the internal table. In the event of failure, no modules are added to the diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 26762969ef8eba..f8fd48e781d6da 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -25,16 +25,10 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyImport_AppendInittab` * :c:func:`PyImport_ExtendInittab` - * :c:func:`PyInitFrozenExtensions` + * :c:func:`!PyInitFrozenExtensions` * :c:func:`PyMem_SetAllocator` * :c:func:`PyMem_SetupDebugHooks` * :c:func:`PyObject_SetArenaAllocator` - * :c:func:`Py_SetPath` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`Py_SetStandardStreamEncoding` - * :c:func:`PySys_AddWarnOption` - * :c:func:`PySys_AddXOption` * :c:func:`PySys_ResetWarnOptions` * Informative functions: @@ -65,7 +59,7 @@ The following functions can be safely called before Python is initialized: :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`. + and :c:func:`Py_GetProgramName`. .. _global-conf-vars: @@ -93,7 +87,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_DebugFlag @@ -107,7 +101,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_DontWriteBytecodeFlag @@ -121,7 +115,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_FrozenFlag @@ -134,7 +128,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_HashRandomizationFlag @@ -149,7 +143,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_IgnoreEnvironmentFlag @@ -157,12 +151,12 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. :c:member:`PyConfig.use_environment` should be used instead, see :ref:`Python Initialization Configuration `. - Ignore all :envvar:`PYTHON*` environment variables, e.g. + Ignore all :envvar:`!PYTHON*` environment variables, e.g. :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. Set by the :option:`-E` and :option:`-I` options. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_InspectFlag @@ -177,7 +171,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_InteractiveFlag @@ -202,7 +196,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_LegacyWindowsFSEncodingFlag @@ -221,7 +215,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_LegacyWindowsStdioFlag @@ -230,7 +224,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. :ref:`Python Initialization Configuration `. If the flag is non-zero, use :class:`io.FileIO` instead of - :class:`WindowsConsoleIO` for :mod:`sys` standard streams. + :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment variable is set to a non-empty string. @@ -239,7 +233,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_NoSiteFlag @@ -254,7 +248,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_NoUserSiteDirectory @@ -268,7 +262,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_OptimizeFlag @@ -279,7 +273,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_QuietFlag @@ -293,7 +287,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_UnbufferedStdioFlag @@ -306,7 +300,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 .. c:var:: int Py_VerboseFlag @@ -322,7 +316,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment variable. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.14 Initializing and finalizing the interpreter @@ -332,16 +326,12 @@ Initializing and finalizing the interpreter .. c:function:: void Py_Initialize() .. index:: - single: Py_SetProgramName() - single: PyEval_InitThreads() single: modules (in module sys) single: path (in module sys) pair: module; builtins pair: module; __main__ pair: module; sys triple: module; search; path - single: PySys_SetArgv() - single: PySys_SetArgvEx() single: Py_FinalizeEx() Initialize the Python interpreter. In an application embedding Python, @@ -352,7 +342,9 @@ Initializing and finalizing the interpreter the table of loaded modules (``sys.modules``), and creates the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use - :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time + the new :c:type:`PyConfig` API of the :ref:`Python Initialization + Configuration ` for that. This is a no-op when called for a + second time (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal error if the initialization fails. @@ -381,6 +373,14 @@ Initializing and finalizing the interpreter :c:func:`Py_Initialize` is called again. +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down `. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + .. c:function:: int Py_FinalizeEx() Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of @@ -401,7 +401,7 @@ Initializing and finalizing the interpreter the application. **Bugs and caveats:** The destruction of modules and objects in modules is done - in random order; this may cause destructors (:meth:`__del__` methods) to fail + in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail when they depend on other objects (even functions) or modules. Dynamically loaded extension modules loaded by Python are not unloaded. Small amounts of memory allocated by the Python interpreter may not be freed (if you find a leak, @@ -425,76 +425,9 @@ Process-wide parameters ======================= -.. c:function:: int Py_SetStandardStreamEncoding(const char *encoding, const char *errors) - - .. index:: - single: Py_Initialize() - single: main() - triple: stdin; stdout; sdterr - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors` - should be used instead, see :ref:`Python Initialization Configuration - `. - - This function should be called before :c:func:`Py_Initialize`, if it is - called at all. It specifies which encoding and error handling to use - with standard IO, with the same meanings as in :func:`str.encode`. - - It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code - to control IO encoding when the environment variable does not work. - - *encoding* and/or *errors* may be ``NULL`` to use - :envvar:`PYTHONIOENCODING` and/or default values (depending on other - settings). - - Note that :data:`sys.stderr` always uses the "backslashreplace" error - handler, regardless of this (or any other) setting. - - If :c:func:`Py_FinalizeEx` is called, this function will need to be called - again in order to affect subsequent calls to :c:func:`Py_Initialize`. - - Returns ``0`` if successful, a nonzero value on error (e.g. calling after the - interpreter has already been initialized). +.. c:function:: wchar_t* Py_GetProgramName() - .. versionadded:: 3.4 - - .. deprecated:: 3.11 - - -.. c:function:: void Py_SetProgramName(const wchar_t *name) - - .. index:: - single: Py_Initialize() - single: main() - single: Py_GetPath() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python - Initialization Configuration `. - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the value - of the ``argv[0]`` argument to the :c:func:`main` function of the program - (converted to wide characters). - This is used by :c:func:`Py_GetPath` and some other functions below to find - the Python run-time libraries relative to the interpreter executable. The - default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - .. deprecated:: 3.11 - - -.. c:function:: wchar* Py_GetProgramName() - - .. index:: single: Py_SetProgramName() - - Return the program name set with :c:func:`Py_SetProgramName`, or the default. + Return the program name set with :c:member:`PyConfig.program_name`, or the default. The returned string points into static storage; the caller should not modify its value. @@ -504,12 +437,15 @@ Process-wide parameters .. versionchanged:: 3.10 It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. deprecated-removed:: 3.13 3.15 + Get :data:`sys.executable` instead. + .. c:function:: wchar_t* Py_GetPrefix() Return the *prefix* for installed platform-independent files. This is derived through a number of complicated rules from the program name set with - :c:func:`Py_SetProgramName` and some environment variables; for example, if the + :c:member:`PyConfig.program_name` and some environment variables; for example, if the program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The returned string points into static storage; the caller should not modify its value. This corresponds to the :makevar:`prefix` variable in the top-level @@ -523,12 +459,15 @@ Process-wide parameters .. versionchanged:: 3.10 It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. deprecated-removed:: 3.13 3.15 + Get :data:`sys.prefix` instead. + .. c:function:: wchar_t* Py_GetExecPrefix() Return the *exec-prefix* for installed platform-*dependent* files. This is derived through a number of complicated rules from the program name set with - :c:func:`Py_SetProgramName` and some environment variables; for example, if the + :c:member:`PyConfig.program_name` and some environment variables; for example, if the program name is ``'/usr/local/bin/python'``, the exec-prefix is ``'/usr/local'``. The returned string points into static storage; the caller should not modify its value. This corresponds to the :makevar:`exec_prefix` @@ -564,16 +503,18 @@ Process-wide parameters .. versionchanged:: 3.10 It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. deprecated-removed:: 3.13 3.15 + Get :data:`sys.exec_prefix` instead. + .. c:function:: wchar_t* Py_GetProgramFullPath() .. index:: - single: Py_SetProgramName() single: executable (in module sys) Return the full program name of the Python executable; this is computed as a side-effect of deriving the default module search path from the program name - (set by :c:func:`Py_SetProgramName` above). The returned string points into + (set by :c:member:`PyConfig.program_name`). The returned string points into static storage; the caller should not modify its value. The value is available to Python code as ``sys.executable``. @@ -583,16 +524,18 @@ Process-wide parameters .. versionchanged:: 3.10 It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. deprecated-removed:: 3.13 3.15 + Get :data:`sys.executable` instead. + .. c:function:: wchar_t* Py_GetPath() .. index:: triple: module; search; path single: path (in module sys) - single: Py_SetPath() Return the default module search path; this is computed from the program name - (set by :c:func:`Py_SetProgramName` above) and some environment variables. + (set by :c:member:`PyConfig.program_name`) and some environment variables. The returned string consists of a series of directory names separated by a platform dependent delimiter character. The delimiter character is ``':'`` on Unix and macOS, ``';'`` on Windows. The returned string points into @@ -609,43 +552,8 @@ Process-wide parameters .. versionchanged:: 3.10 It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - -.. c:function:: void Py_SetPath(const wchar_t *) - - .. index:: - triple: module; search; path - single: path (in module sys) - single: Py_GetPath() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.module_search_paths` and - :c:member:`PyConfig.module_search_paths_set` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set the default module search path. If this function is called before - :c:func:`Py_Initialize`, then :c:func:`Py_GetPath` won't attempt to compute a - default search path but uses the one provided instead. This is useful if - Python is embedded by an application that has full knowledge of the location - of all modules. The path components should be separated by the platform - dependent delimiter character, which is ``':'`` on Unix and macOS, ``';'`` - on Windows. - - This also causes :data:`sys.executable` to be set to the program - full path (see :c:func:`Py_GetProgramFullPath`) and for :data:`sys.prefix` and - :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these - if required after calling :c:func:`Py_Initialize`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - The path argument is copied internally, so the caller may free it after the - call completes. - - .. versionchanged:: 3.8 - The program full path is now used for :data:`sys.executable`, instead - of the program name. - - .. deprecated:: 3.11 + .. deprecated-removed:: 3.13 3.15 + Get :data:`sys.path` instead. .. c:function:: const char* Py_GetVersion() @@ -718,110 +626,10 @@ Process-wide parameters ``sys.version``. -.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) - - .. index:: - single: main() - single: Py_FatalError() - single: argv (in module sys) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and - :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set :data:`sys.argv` based on *argc* and *argv*. These parameters are - similar to those passed to the program's :c:func:`main` function with the - difference that the first entry should refer to the script file to be - executed rather than the executable hosting the Python interpreter. If there - isn't a script that will be run, the first entry in *argv* can be an empty - string. If this function fails to initialize :data:`sys.argv`, a fatal - condition is signalled using :c:func:`Py_FatalError`. - - If *updatepath* is zero, this is all the function does. If *updatepath* - is non-zero, the function also modifies :data:`sys.path` according to the - following algorithm: - - - If the name of an existing script is passed in ``argv[0]``, the absolute - path of the directory where the script is located is prepended to - :data:`sys.path`. - - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point - to an existing file name), an empty string is prepended to - :data:`sys.path`, which is the same as prepending the current working - directory (``"."``). - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. note:: - It is recommended that applications embedding the Python interpreter - for purposes other than executing a single script pass ``0`` as *updatepath*, - and update :data:`sys.path` themselves if desired. - See `CVE-2008-5983 `_. - - On versions before 3.1.3, you can achieve the same effect by manually - popping the first :data:`sys.path` element after having called - :c:func:`PySys_SetArgv`, for example using:: - - PyRun_SimpleString("import sys; sys.path.pop(0)\n"); - - .. versionadded:: 3.1.3 - - .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; - check w/ Guido. - - .. deprecated:: 3.11 - - -.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used - instead, see :ref:`Python Initialization Configuration `. - - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to ``1`` unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - - .. deprecated:: 3.11 - - -.. c:function:: void Py_SetPythonHome(const wchar_t *home) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.home` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set the default "home" directory, that is, the location of the standard - Python libraries. See :envvar:`PYTHONHOME` for the meaning of the - argument string. - - The argument should point to a zero-terminated character string in static - storage whose contents will not change for the duration of the program's - execution. No code in the Python interpreter will change the contents of - this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - .. deprecated:: 3.11 - - .. c:function:: wchar_t* Py_GetPythonHome() - Return the default "home", that is, the value set by a previous call to - :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME` + Return the default "home", that is, the value set by + :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME` environment variable if it is set. This function should not be called before :c:func:`Py_Initialize`, otherwise @@ -830,6 +638,10 @@ Process-wide parameters .. versionchanged:: 3.10 It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. deprecated-removed:: 3.13 3.15 + Get :c:member:`PyConfig.home` or :envvar:`PYTHONHOME` environment + variable instead. + .. _threads: @@ -981,7 +793,7 @@ the fork, and releasing them afterwards. In addition, it resets any :ref:`lock-objects` in the child. When extending or embedding Python, there is no way to inform Python of additional (non-Python) locks that need to be acquired before or reset after a fork. OS facilities such as -:c:func:`pthread_atfork` would need to be used to accomplish the same thing. +:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. Additionally, when extending or embedding Python, calling :c:func:`fork` directly rather than through :func:`os.fork` (and returning to or calling into Python) may result in a deadlock by one of Python's internal locks @@ -1023,47 +835,11 @@ code, or when embedding the Python interpreter: .. c:type:: PyThreadState This data structure represents the state of a single thread. The only public - data member is :attr:`interp` (:c:expr:`PyInterpreterState *`), which points to - this thread's interpreter state. + data member is: + .. c:member:: PyInterpreterState *interp -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: int PyEval_ThreadsInitialized() - - Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This - function can be called without holding the GIL, and therefore can be used to - avoid calls to the locking API when running single-threaded. - - .. versionchanged:: 3.7 - The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`. - - .. deprecated:: 3.9 + This thread's interpreter state. .. c:function:: PyThreadState* PyEval_SaveThread() @@ -1084,7 +860,7 @@ code, or when embedding the Python interpreter: .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. @@ -1094,6 +870,19 @@ code, or when embedding the Python interpreter: When the current thread state is ``NULL``, this issues a fatal error (so that the caller needn't check for ``NULL``). + See also :c:func:`PyThreadState_GetUnchecked`. + + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + .. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) @@ -1130,7 +919,7 @@ with sub-interpreters: .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. @@ -1396,7 +1185,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. function does not steal any references to *exc*. To prevent naive misuse, you must write your own C extension to call this. Must be called with the GIL held. Returns the number of thread states modified; this is normally one, but will be - zero if the thread id isn't found. If *exc* is :const:`NULL`, the pending + zero if the thread id isn't found. If *exc* is ``NULL``, the pending exception (if any) for the thread is cleared. This raises no exceptions. .. versionchanged:: 3.7 @@ -1412,7 +1201,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. @@ -1437,39 +1226,6 @@ All of the following functions must be called after :c:func:`Py_Initialize`. available (even when threads have not been initialized). -.. c:function:: void PyEval_AcquireLock() - - Acquire the global interpreter lock. The lock must have been created earlier. - If this thread already has the lock, a deadlock ensues. - - .. deprecated:: 3.2 - This function does not update the current thread state. Please use - :c:func:`PyEval_RestoreThread` or :c:func:`PyEval_AcquireThread` - instead. - - .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - -.. c:function:: void PyEval_ReleaseLock() - - Release the global interpreter lock. The lock must have been created earlier. - - .. deprecated:: 3.2 - This function does not update the current thread state. Please use - :c:func:`PyEval_SaveThread` or :c:func:`PyEval_ReleaseThread` - instead. - - .. _sub-interpreter-support: Sub-interpreter support @@ -1491,7 +1247,96 @@ You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` function. You can create and destroy them using the following functions: -.. c:function:: PyThreadState* Py_NewInterpreter() +.. c:type:: PyInterpreterConfig + + Structure containing most parameters to configure a sub-interpreter. + Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and + never modified by the runtime. + + .. versionadded:: 3.12 + + Structure fields: + + .. c:member:: int use_main_obmalloc + + If this is ``0`` then the sub-interpreter will use its own + "object" allocator state. + Otherwise it will use (share) the main interpreter's. + + If this is ``0`` then + :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` + must be ``1`` (non-zero). + If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` + must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. + + .. c:member:: int allow_fork + + If this is ``0`` then the runtime will not support forking the + process in any thread where the sub-interpreter is currently active. + Otherwise fork is unrestricted. + + Note that the :mod:`subprocess` module still works + when fork is disallowed. + + .. c:member:: int allow_exec + + If this is ``0`` then the runtime will not support replacing the + current process via exec (e.g. :func:`os.execv`) in any thread + where the sub-interpreter is currently active. + Otherwise exec is unrestricted. + + Note that the :mod:`subprocess` module still works + when exec is disallowed. + + .. c:member:: int allow_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create threads. + Otherwise threads are allowed. + + .. c:member:: int allow_daemon_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create daemon threads. + Otherwise daemon threads are allowed (as long as + :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). + + .. c:member:: int check_multi_interp_extensions + + If this is ``0`` then all extension modules may be imported, + including legacy (single-phase init) modules, + in any thread where the sub-interpreter is currently active. + Otherwise only multi-phase init extension modules + (see :pep:`489`) may be imported. + (Also see :c:macro:`Py_mod_multiple_interpreters`.) + + This must be ``1`` (non-zero) if + :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. + + .. c:member:: int gil + + This determines the operation of the GIL for the sub-interpreter. + It may be one of the following: + + .. c:namespace:: NULL + + .. c:macro:: PyInterpreterConfig_DEFAULT_GIL + + Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). + + .. c:macro:: PyInterpreterConfig_SHARED_GIL + + Use (share) the main interpreter's GIL. + + .. c:macro:: PyInterpreterConfig_OWN_GIL + + Use the sub-interpreter's own GIL. + + If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then + :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. + + +.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) .. index:: pair: module; builtins @@ -1511,16 +1356,47 @@ function. You can create and destroy them using the following functions: ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying file descriptors). - The return value points to the first thread state created in the new + The given *config* controls the options with which the interpreter + is initialized. + + Upon success, *tstate_p* will be set to the first thread state + created in the new sub-interpreter. This thread state is made in the current thread state. Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, ``NULL`` is - returned; no exception is set since the exception state is stored in the - current thread state and there may not be a current thread state. (Like all - other Python/C API functions, the global interpreter lock must be held before - calling this function and is still held when it returns; however, unlike most - other Python/C API functions, there needn't be a current thread state on - entry.) + below. If creation of the new interpreter is unsuccessful, + *tstate_p* is set to ``NULL``; + no exception is set since the exception state is stored in the + current thread state and there may not be a current thread state. + + Like all other Python/C API functions, the global interpreter lock + must be held before calling this function and is still held when it + returns. Likewise a current thread state must be set on entry. On + success, the returned thread state will be set as current. If the + sub-interpreter is created with its own GIL then the GIL of the + calling interpreter will be released. When the function returns, + the new interpreter's GIL will be held by the current thread and + the previously interpreter's GIL will remain released here. + + .. versionadded:: 3.12 + + Sub-interpreters are most effective when isolated from each other, + with certain functionality restricted:: + + PyInterpreterConfig config = { + .use_main_obmalloc = 0, + .allow_fork = 0, + .allow_exec = 0, + .allow_threads = 1, + .allow_daemon_threads = 0, + .check_multi_interp_extensions = 1, + .gil = PyInterpreterConfig_OWN_GIL, + }; + PyThreadState *tstate = Py_NewInterpreterFromConfig(&config); + + Note that the config is used only briefly and does not get modified. + During initialization the config's values are converted into various + :c:type:`PyInterpreterState` values. A read-only copy of the config + may be stored internally on the :c:type:`PyInterpreterState`. .. index:: single: Py_FinalizeEx() @@ -1555,19 +1431,79 @@ function. You can create and destroy them using the following functions: .. index:: single: close() (in module os) +.. c:function:: PyThreadState* Py_NewInterpreter(void) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is essentially just a wrapper + around :c:func:`Py_NewInterpreterFromConfig` with a config that + preserves the existing behavior. The result is an unisolated + sub-interpreter that shares the main interpreter's GIL, allows + fork/exec, allows daemon threads, and allows single-phase init + modules. + + .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) .. index:: single: Py_FinalizeEx() - Destroy the (sub-)interpreter represented by the given thread state. The given - thread state must be the current thread state. See the discussion of thread - states below. When the call returns, the current thread state is ``NULL``. All - thread states associated with this interpreter are destroyed. (The global - interpreter lock must be held before calling this function and is still held - when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that + Destroy the (sub-)interpreter represented by the given thread state. + The given thread state must be the current thread state. See the + discussion of thread states below. When the call returns, + the current thread state is ``NULL``. All thread states associated + with this interpreter are destroyed. The global interpreter lock + used by the target interpreter must be held before calling this + function. No GIL is held when it returns. + + :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that haven't been explicitly destroyed at that point. +A Per-Interpreter GIL +--------------------- + +Using :c:func:`Py_NewInterpreterFromConfig` you can create +a sub-interpreter that is completely isolated from other interpreters, +including having its own GIL. The most important benefit of this +isolation is that such an interpreter can execute Python code without +being blocked by other interpreters or blocking any others. Thus a +single Python process can truly take advantage of multiple CPU cores +when running Python code. The isolation also encourages a different +approach to concurrency than that of just using threads. +(See :pep:`554`.) + +Using an isolated interpreter requires vigilance in preserving that +isolation. That especially means not sharing any objects or mutable +state without guarantees about thread-safety. Even objects that are +otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared +because of the refcount. One simple but less-efficient approach around +this is to use a global lock around all use of some state (or object). +Alternately, effectively immutable objects (like integers or strings) +can be made safe in spite of their refcounts by making them :term:`immortal`. +In fact, this has been done for the builtin singletons, small integers, +and a number of other builtin objects. + +If you preserve isolation then you will have access to proper multi-core +computing without the complications that come with free-threading. +Failure to preserve isolation will expose you to the full consequences +of free-threading, including races and hard-to-debug crashes. + +Aside from that, one of the main challenges of using multiple isolated +interpreters is how to communicate between them safely (not break +isolation) and efficiently. The runtime and stdlib do not provide +any standard approach to this yet. A future stdlib module would help +mitigate the effort of preserving isolation and expose effective tools +for communicating (and sharing) data between interpreters. + +.. versionadded:: 3.12 + + Bugs and caveats ---------------- @@ -1675,32 +1611,32 @@ Python-level trace functions in previous versions. The type of the trace function registered using :c:func:`PyEval_SetProfile` and :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the registration function as *obj*, *frame* is the frame object to which the event - pertains, *what* is one of the constants :const:`PyTrace_CALL`, - :const:`PyTrace_EXCEPTION`, :const:`PyTrace_LINE`, :const:`PyTrace_RETURN`, - :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, :const:`PyTrace_C_RETURN`, - or :const:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - - +------------------------------+----------------------------------------+ - | Value of *what* | Meaning of *arg* | - +==============================+========================================+ - | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +------------------------------+----------------------------------------+ - | :const:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +------------------------------+----------------------------------------+ - | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +------------------------------+----------------------------------------+ - | :const:`PyTrace_RETURN` | Value being returned to the caller, | - | | or ``NULL`` if caused by an exception. | - +------------------------------+----------------------------------------+ - | :const:`PyTrace_C_CALL` | Function object being called. | - +------------------------------+----------------------------------------+ - | :const:`PyTrace_C_EXCEPTION` | Function object being called. | - +------------------------------+----------------------------------------+ - | :const:`PyTrace_C_RETURN` | Function object being called. | - +------------------------------+----------------------------------------+ - | :const:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +------------------------------+----------------------------------------+ + pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, + :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, + :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, + or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: + + +-------------------------------+----------------------------------------+ + | Value of *what* | Meaning of *arg* | + +===============================+========================================+ + | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | + | | or ``NULL`` if caused by an exception. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_CALL` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_RETURN` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ .. c:var:: int PyTrace_CALL @@ -1726,7 +1662,8 @@ Python-level trace functions in previous versions. The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`f_trace_lines` to *0* on that frame. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. .. c:var:: int PyTrace_RETURN @@ -1758,7 +1695,7 @@ Python-level trace functions in previous versions. The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not profiling functions) when a new opcode is about to be executed. This event is not emitted by default: it must be explicitly requested by setting - :attr:`f_trace_opcodes` to *1* on the frame. + :attr:`~frame.f_trace_opcodes` to *1* on the frame. .. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) @@ -1767,8 +1704,8 @@ Python-level trace functions in previous versions. function as its first parameter, and may be any Python object, or ``NULL``. If the profile function needs to maintain state, using a different value for *obj* for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :const:`PyTrace_LINE` - :const:`PyTrace_OPCODE` and :const:`PyTrace_EXCEPTION`. + profile function is called for all monitored events except :c:data:`PyTrace_LINE` + :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. See also the :func:`sys.setprofile` function. @@ -1793,8 +1730,8 @@ Python-level trace functions in previous versions. :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number events and per-opcode events, but does not receive any event related to C function objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` - will not receive :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or - :const:`PyTrace_C_RETURN` as a value for the *what* parameter. + will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or + :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. See also the :func:`sys.settrace` function. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 161def0b4baf3a..47a8fbb2cd9c97 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -82,6 +82,8 @@ PyWideStringList If *length* is non-zero, *items* must be non-``NULL`` and all strings must be non-``NULL``. + .. c:namespace:: NULL + Methods: .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) @@ -101,6 +103,8 @@ PyWideStringList Python must be preinitialized to call this function. + .. c:namespace:: PyWideStringList + Structure fields: .. c:member:: Py_ssize_t length @@ -135,6 +139,8 @@ PyStatus Name of the function which created an error, can be ``NULL``. + .. c:namespace:: NULL + Functions to create a status: .. c:function:: PyStatus PyStatus_Ok(void) @@ -210,6 +216,8 @@ PyPreConfig Structure used to preinitialize Python. + .. c:namespace:: NULL + Function to initialize a preconfiguration: .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) @@ -222,6 +230,8 @@ PyPreConfig Initialize the preconfiguration with :ref:`Isolated Configuration `. + .. c:namespace:: PyPreConfig + Structure fields: .. c:member:: int allocator @@ -243,11 +253,21 @@ PyPreConfig * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc memory allocator ` with :ref:`debug hooks `. + * ``PYMEM_ALLOCATOR_MIMALLOC`` (``6``): use ``mimalloc``, a fast + malloc replacement. + * ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` (``7``): use ``mimalloc``, a fast + malloc replacement with :ref:`debug hooks `. + ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` are not supported if Python is :option:`configured using --without-pymalloc <--without-pymalloc>`. + ``PYMEM_ALLOCATOR_MIMALLOC`` and ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` are + not supported if Python is :option:`configured using --without-mimalloc + <--without-mimalloc>` or if the underlying atomic support isn't + available. + See :ref:`Memory Management `. Default: ``PYMEM_ALLOCATOR_NOT_SET``. @@ -429,6 +449,8 @@ PyConfig When done, the :c:func:`PyConfig_Clear` function must be used to release the configuration memory. + .. c:namespace:: NULL + Structure methods: .. c:function:: void PyConfig_InitPythonConfig(PyConfig *config) @@ -522,16 +544,28 @@ PyConfig Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` is used, this method must be called before other methods, since the preinitialization configuration depends on command line arguments (if - :c:member:`parse_argv` is non-zero). + :c:member:`~PyConfig.parse_argv` is non-zero). The caller of these methods is responsible to handle exceptions (error or exit) using ``PyStatus_Exception()`` and ``Py_ExitStatusException()``. + .. c:namespace:: PyConfig + Structure fields: .. c:member:: PyWideStringList argv - Command line arguments: :data:`sys.argv`. + .. index:: + single: main() + single: argv (in module sys) + + Set :data:`sys.argv` command line arguments based on + :c:member:`~PyConfig.argv`. These parameters are similar to those passed + to the program's :c:func:`main` function with the difference that the + first entry should refer to the script file to be executed rather than + the executable hosting the Python interpreter. If there isn't a script + that will be run, the first entry in :c:member:`~PyConfig.argv` can be an + empty string. Set :c:member:`~PyConfig.parse_argv` to ``1`` to parse :c:member:`~PyConfig.argv` the same way the regular Python parses Python @@ -572,6 +606,8 @@ PyConfig Part of the :ref:`Python Path Configuration ` output. + See also :c:member:`PyConfig.exec_prefix`. + .. c:member:: wchar_t* base_executable Python base executable: :data:`sys._base_executable`. @@ -584,6 +620,8 @@ PyConfig Part of the :ref:`Python Path Configuration ` output. + See also :c:member:`PyConfig.executable`. + .. c:member:: wchar_t* base_prefix :data:`sys.base_prefix`. @@ -592,6 +630,8 @@ PyConfig Part of the :ref:`Python Path Configuration ` output. + See also :c:member:`PyConfig.prefix`. + .. c:member:: int buffered_stdio If equals to ``0`` and :c:member:`~PyConfig.configure_c_stdio` is non-zero, @@ -686,7 +726,7 @@ PyConfig Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable. - Need a special build of Python with the ``Py_TRACE_REFS`` macro defined: + Needs a special build of Python with the ``Py_TRACE_REFS`` macro defined: see the :option:`configure --with-trace-refs option <--with-trace-refs>`. Default: ``0``. @@ -700,6 +740,8 @@ PyConfig Part of the :ref:`Python Path Configuration ` output. + See also :c:member:`PyConfig.base_exec_prefix`. + .. c:member:: wchar_t* executable The absolute path of the executable binary for the Python interpreter: @@ -709,6 +751,8 @@ PyConfig Part of the :ref:`Python Path Configuration ` output. + See also :c:member:`PyConfig.base_executable`. + .. c:member:: int faulthandler Enable faulthandler? @@ -780,10 +824,8 @@ PyConfig .. c:member:: wchar_t* home - Python home directory. - - If :c:func:`Py_SetPythonHome` has been called, use its argument if it is - not ``NULL``. + Set the default Python "home" directory, that is, the location of the + standard Python libraries (see :envvar:`PYTHONHOME`). Set by the :envvar:`PYTHONHOME` environment variable. @@ -846,6 +888,19 @@ PyConfig .. versionadded:: 3.12 + .. c:member:: int cpu_count + + If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will + override the return values of :func:`os.cpu_count`, + :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. + + Configured by the :samp:`-X cpu_count={n|default}` command line + flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. + + Default: ``-1``. + + .. versionadded:: 3.13 + .. c:member:: int isolated If greater than ``0``, enable isolated mode: @@ -871,7 +926,7 @@ PyConfig .. c:member:: int legacy_windows_stdio If non-zero, use :class:`io.FileIO` instead of - :class:`io.WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` + :class:`!io._WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr`. Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment @@ -920,7 +975,7 @@ PyConfig .. c:member:: wchar_t* pythonpath_env Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` - (:data:`os.path.pathsep`). + (:data:`os.pathsep`). Set by the :envvar:`PYTHONPATH` environment variable. @@ -1003,7 +1058,7 @@ PyConfig Incremented by the :option:`-d` command line option. Set to the :envvar:`PYTHONDEBUG` environment variable value. - Need a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro + Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro must be defined). Default: ``0``. @@ -1029,12 +1084,13 @@ PyConfig Part of the :ref:`Python Path Configuration ` output. + See also :c:member:`PyConfig.base_prefix`. + .. c:member:: wchar_t* program_name Program name used to initialize :c:member:`~PyConfig.executable` and in early error messages during Python initialization. - * If :func:`Py_SetProgramName` has been called, use its argument. * On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set. * If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use :envvar:`__PYVENV_LAUNCHER__` environment variable if set. @@ -1054,6 +1110,7 @@ PyConfig Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and the :envvar:`PYTHONPYCACHEPREFIX` environment variable. + The command-line option takes precedence. If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. @@ -1097,13 +1154,27 @@ PyConfig Default: ``NULL``. + .. c:member:: wchar_t* run_presite + + ``package.module`` path to module that should be imported before + ``site.py`` is run. + + Set by the :option:`-X presite=package.module <-X>` command-line + option and the :envvar:`PYTHON_PRESITE` environment variable. + The command-line option takes precedence. + + Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro + must be defined). + + Default: ``NULL``. + .. c:member:: int show_ref_count - Show total reference count at exit? + Show total reference count at exit (excluding :term:`immortal` objects)? Set to ``1`` by :option:`-X showrefcount <-X>` command line option. - Need a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` + Needs a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` macro must be defined). Default: ``0``. @@ -1120,7 +1191,7 @@ PyConfig Set to ``0`` by the :option:`-S` command line option. - :data:`sys.flags.no_site` is set to the inverted value of + :data:`sys.flags.no_site ` is set to the inverted value of :c:member:`~PyConfig.site_import`. Default: ``1``. @@ -1144,9 +1215,6 @@ PyConfig :data:`sys.stderr` (but :data:`sys.stderr` always uses ``"backslashreplace"`` error handler). - If :c:func:`Py_SetStandardStreamEncoding` has been called, use its - *error* and *errors* arguments if they are not ``NULL``. - Use the :envvar:`PYTHONIOENCODING` environment variable if it is non-empty. @@ -1162,6 +1230,8 @@ PyConfig or if the LC_CTYPE locale is "C" or "POSIX". * ``"strict"`` otherwise. + See also :c:member:`PyConfig.legacy_windows_stdio`. + .. c:member:: int tracemalloc Enable tracemalloc? @@ -1509,7 +1579,7 @@ If a ``._pth`` file is present: * Set :c:member:`~PyConfig.safe_path` to ``1``. The ``__PYVENV_LAUNCHER__`` environment variable is used to set -:c:member:`PyConfig.base_executable` +:c:member:`PyConfig.base_executable`. Py_RunMain() diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 8de76e55cd0586..4dbca92b18b5cd 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -105,6 +105,30 @@ defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`). Others of a more general utility are defined here. This is not necessarily a complete listing. +.. c:macro:: PyMODINIT_FUNC + + Declare an extension module ``PyInit`` initialization function. The function + return type is :c:expr:`PyObject*`. The macro declares any special linkage + declarations required by the platform, and for C++ declares the function as + ``extern "C"``. + + The initialization function must be named :samp:`PyInit_{name}`, where + *name* is the name of the module, and should be the only non-\ ``static`` + item defined in the module file. Example:: + + static struct PyModuleDef spam_module = { + PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModule_Create(&spam_module); + } + + .. c:macro:: Py_ABS(x) Return the absolute value of ``x``. @@ -287,52 +311,58 @@ true if (and only if) the object pointed to by *a* is a Python list. Reference Counts ---------------- -The reference count is important because today's computers have a finite (and -often severely limited) memory size; it counts how many different places there -are that have a reference to an object. Such a place could be another object, -or a global (or static) C variable, or a local variable in some C function. -When an object's reference count becomes zero, the object is deallocated. If -it contains references to other objects, their reference count is decremented. -Those other objects may be deallocated in turn, if this decrement makes their -reference count become zero, and so on. (There's an obvious problem with -objects that reference each other here; for now, the solution is "don't do -that.") +The reference count is important because today's computers have a finite +(and often severely limited) memory size; it counts how many different +places there are that have a :term:`strong reference` to an object. +Such a place could be another object, or a global (or static) C variable, +or a local variable in some C function. +When the last :term:`strong reference` to an object is released +(i.e. its reference count becomes zero), the object is deallocated. +If it contains references to other objects, those references are released. +Those other objects may be deallocated in turn, if there are no more +references to them, and so on. (There's an obvious problem with +objects that reference each other here; for now, the solution +is "don't do that.") .. index:: single: Py_INCREF() single: Py_DECREF() -Reference counts are always manipulated explicitly. The normal way is to use -the macro :c:func:`Py_INCREF` to increment an object's reference count by one, -and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro +Reference counts are always manipulated explicitly. The normal way is +to use the macro :c:func:`Py_INCREF` to take a new reference to an +object (i.e. increment its reference count by one), +and :c:func:`Py_DECREF` to release that reference (i.e. decrement the +reference count by one). The :c:func:`Py_DECREF` macro is considerably more complex than the incref one, since it must check whether the reference count becomes zero and then cause the object's deallocator to be -called. The deallocator is a function pointer contained in the object's type -structure. The type-specific deallocator takes care of decrementing the -reference counts for other objects contained in the object if this is a compound +called. The deallocator is a function pointer contained in the object's type +structure. The type-specific deallocator takes care of releasing references +for other objects contained in the object if this is a compound object type, such as a list, as well as performing any additional finalization that's needed. There's no chance that the reference count can overflow; at least as many bits are used to hold the reference count as there are distinct memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). Thus, the reference count increment is a simple operation. -It is not necessary to increment an object's reference count for every local -variable that contains a pointer to an object. In theory, the object's +It is not necessary to hold a :term:`strong reference` (i.e. increment +the reference count) for every local variable that contains a pointer +to an object. In theory, the object's reference count goes up by one when the variable is made to point to it and it goes down by one when the variable goes out of scope. However, these two cancel each other out, so at the end the reference count hasn't changed. The only real reason to use the reference count is to prevent the object from being deallocated as long as our variable is pointing to it. If we know that there is at least one other reference to the object that lives at least as long as -our variable, there is no need to increment the reference count temporarily. +our variable, there is no need to take a new :term:`strong reference` +(i.e. increment the reference count) temporarily. An important situation where this arises is in objects that are passed as arguments to C functions in an extension module that are called from Python; the call mechanism guarantees to hold a reference to every argument for the duration of the call. However, a common pitfall is to extract an object from a list and hold on to it -for a while without incrementing its reference count. Some other operation might -conceivably remove the object from the list, decrementing its reference count +for a while without taking a new reference. Some other operation might +conceivably remove the object from the list, releasing that reference, and possibly deallocating it. The real danger is that innocent-looking operations may invoke arbitrary Python code which could do this; there is a code path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so @@ -340,7 +370,8 @@ almost any operation is potentially dangerous. A safe approach is to always use the generic operations (functions whose name begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). -These operations always increment the reference count of the object they return. +These operations always create a new :term:`strong reference` +(i.e. increment the reference count) of the object they return. This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when they are done with the result; this soon becomes second nature. @@ -356,7 +387,7 @@ to objects (objects are not owned: they are always shared). "Owning a reference" means being responsible for calling Py_DECREF on it when the reference is no longer needed. Ownership can also be transferred, meaning that the code that receives ownership of the reference then becomes responsible for -eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` +eventually releasing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` when it's no longer needed---or passing on this responsibility (usually to its caller). When a function passes ownership of a reference on to its caller, the caller is said to receive a *new* reference. When no ownership is transferred, @@ -414,9 +445,9 @@ For example, the above two blocks of code could be replaced by the following It is much more common to use :c:func:`PyObject_SetItem` and friends with items whose references you are only borrowing, like arguments that were passed in to -the function you are writing. In that case, their behaviour regarding reference -counts is much saner, since you don't have to increment a reference count so you -can give a reference away ("have it be stolen"). For example, this function +the function you are writing. In that case, their behaviour regarding references +is much saner, since you don't have to take a new reference just so you +can give that reference away ("have it be stolen"). For example, this function sets all items of a list (actually, any mutable sequence) to a given item:: int @@ -616,7 +647,7 @@ and lose important information about the exact cause of the error. .. index:: single: sum_sequence() A simple example of detecting exceptions and passing them on is shown in the -:c:func:`sum_sequence` example above. It so happens that this example doesn't +:c:func:`!sum_sequence` example above. It so happens that this example doesn't need to clean up any owned references when it detects an error. The following example function shows some error cleanup. First, to remind you why you like Python, we show the equivalent Python code:: @@ -739,14 +770,14 @@ environment variable :envvar:`PYTHONHOME`, or insert additional directories in front of the standard path by setting :envvar:`PYTHONPATH`. .. index:: - single: Py_SetProgramName() single: Py_GetPath() single: Py_GetPrefix() single: Py_GetExecPrefix() single: Py_GetProgramFullPath() -The embedding application can steer the search by calling -``Py_SetProgramName(file)`` *before* calling :c:func:`Py_Initialize`. Note that +The embedding application can steer the search by setting +:c:member:`PyConfig.program_name` *before* calling +:c:func:`Py_InitializeFromConfig`. Note that :envvar:`PYTHONHOME` still overrides this and :envvar:`PYTHONPATH` is still inserted in front of the standard path. An application that requires total control has to provide its own implementation of :c:func:`Py_GetPath`, diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 3fcf099134d4dd..6b7ba8c9979163 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -6,7 +6,7 @@ Iterator Objects ---------------- Python provides two general-purpose iterator objects. The first, a sequence -iterator, works with an arbitrary sequence supporting the :meth:`__getitem__` +iterator, works with an arbitrary sequence supporting the :meth:`~object.__getitem__` method. The second works with a callable object and a sentinel value, calling the callable for each item in the sequence, and ending the iteration when the sentinel value is returned. @@ -19,7 +19,7 @@ sentinel value is returned. types. -.. c:function:: int PySeqIter_Check(op) +.. c:function:: int PySeqIter_Check(PyObject *op) Return true if the type of *op* is :c:data:`PySeqIter_Type`. This function always succeeds. @@ -38,7 +38,7 @@ sentinel value is returned. two-argument form of the :func:`iter` built-in function. -.. c:function:: int PyCallIter_Check(op) +.. c:function:: int PyCallIter_Check(PyObject *op) Return true if the type of *op* is :c:data:`PyCallIter_Type`. This function always succeeds. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index dbf35611eccd3e..c8b64bad702f50 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -86,6 +86,10 @@ List Objects Macro form of :c:func:`PyList_SetItem` without error checking. This is normally only used to fill in new lists where there is no previous content. + Bounds checking is performed as an assertion if Python is built in + :ref:`debug mode ` or :option:`with assertions + <--with-assertions>`. + .. note:: This macro "steals" a reference to *item*, and, unlike @@ -124,6 +128,30 @@ List Objects list is not supported. +.. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable) + + Extend *list* with the contents of *iterable*. This is the same as + ``PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable)`` + and analogous to ``list.extend(iterable)`` or ``list += iterable``. + + Raise an exception and return ``-1`` if *list* is not a :class:`list` + object. Return 0 on success. + + .. versionadded:: 3.13 + + +.. c:function:: int PyList_Clear(PyObject *list) + + Remove all items from *list*. This is the same as + ``PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL)`` and analogous to + ``list.clear()`` or ``del list[:]``. + + Raise an exception and return ``-1`` if *list* is not a :class:`list` + object. Return 0 on success. + + .. versionadded:: 3.13 + + .. c:function:: int PyList_Sort(PyObject *list) Sort the items of *list* in place. Return ``0`` on success, ``-1`` on diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 4a71c89ad85d31..045604870d3c84 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -121,7 +121,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. single: OverflowError (built-in exception) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. Raise :exc:`OverflowError` if the value of *obj* is out of range for a @@ -130,30 +130,38 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. + + +.. c:function:: int PyLong_AsInt(PyObject *obj) + + Similar to :c:func:`PyLong_AsLong`, but store the result in a C + :c:expr:`int` instead of a C :c:expr:`long`. + + .. versionadded:: 3.13 .. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` + instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. - If the value of *obj* is greater than :const:`LONG_MAX` or less than - :const:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and + If the value of *obj* is greater than :c:macro:`LONG_MAX` or less than + :c:macro:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: long long PyLong_AsLongLong(PyObject *obj) @@ -162,7 +170,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. single: OverflowError (built-in exception) Return a C :c:expr:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. Raise :exc:`OverflowError` if the value of *obj* is out of range for a @@ -171,20 +179,20 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow) Return a C :c:expr:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. - If the value of *obj* is greater than :const:`LLONG_MAX` or less than - :const:`LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, + If the value of *obj* is greater than :c:macro:`LLONG_MAX` or less than + :c:macro:`LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. @@ -193,10 +201,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.2 .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) @@ -267,7 +275,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *obj) Return a C :c:expr:`unsigned long` representation of *obj*. If *obj* is not - an instance of :c:type:`PyLongObject`, first call its :meth:`__index__` + an instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. If the value of *obj* is out of range for an :c:expr:`unsigned long`, @@ -277,17 +285,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj) Return a C :c:expr:`unsigned long long` representation of *obj*. If *obj* is not an instance of :c:type:`PyLongObject`, first call its - :meth:`__index__` method (if present) to convert it to a + :meth:`~object.__index__` method (if present) to convert it to a :c:type:`PyLongObject`. If the value of *obj* is out of range for an :c:expr:`unsigned long long`, @@ -297,10 +305,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. to disambiguate. .. versionchanged:: 3.8 - Use :meth:`__index__` if available. + Use :meth:`~object.__index__` if available. .. versionchanged:: 3.10 - This function will no longer use :meth:`__int__`. + This function will no longer use :meth:`~object.__int__`. .. c:function:: double PyLong_AsDouble(PyObject *pylong) @@ -322,3 +330,27 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. with :c:func:`PyLong_FromVoidPtr`. Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + + +.. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op) + + Return 1 if *op* is compact, 0 otherwise. + + This function makes it possible for performance-critical code to implement + a “fast path†for small integers. For compact values use + :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a + :c:func:`PyLong_As* ` function or + :c:func:`calling ` :meth:`int.to_bytes`. + + The speedup is expected to be negligible for most users. + + Exactly what values are considered compact is an implementation detail + and is subject to change. + +.. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) + + If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`, + return its value. + + Otherwise, the return value is undefined. + diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index cffb0ed50fb77d..1f55c0aa955c75 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -13,7 +13,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and Return ``1`` if the object provides the mapping protocol or supports slicing, and ``0`` otherwise. Note that it returns ``1`` for Python classes with - a :meth:`__getitem__` method, since in general it is impossible to + a :meth:`~object.__getitem__` method, since in general it is impossible to determine what type of keys the class supports. This function always succeeds. @@ -28,52 +28,100 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) - Return element of *o* corresponding to the string *key* or ``NULL`` on failure. - This is the equivalent of the Python expression ``o[key]``. - See also :c:func:`PyObject_GetItem`. + This is the same as :c:func:`PyObject_GetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + +.. c:function:: int PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) + + Variant of :c:func:`PyObject_GetItem` which doesn't raise + :exc:`KeyError` if the key is not found. + + If the key is found, return ``1`` and set *\*result* to a new + :term:`strong reference` to the corresponding value. + If the key is not found, return ``0`` and set *\*result* to ``NULL``; + the :exc:`KeyError` is silenced. + If an error other than :exc:`KeyError` is raised, return ``-1`` and + set *\*result* to ``NULL``. + + .. versionadded:: 3.13 + + +.. c:function:: int PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) + + This is the same as :c:func:`PyMapping_GetOptionalItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: 3.13 .. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v) - Map the string *key* to the value *v* in object *o*. Returns ``-1`` on - failure. This is the equivalent of the Python statement ``o[key] = v``. - See also :c:func:`PyObject_SetItem`. This function *does not* steal a - reference to *v*. + This is the same as :c:func:`PyObject_SetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key) - Remove the mapping for the object *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. This is an alias of :c:func:`PyObject_DelItem`. .. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key) - Remove the mapping for the string *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. + This is the same as :c:func:`PyObject_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. -.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key) +.. c:function:: int PyMapping_HasKeyWithError(PyObject *o, PyObject *key) Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. This is equivalent to the Python expression ``key in o``. - This function always succeeds. + On failure, return ``-1``. - Note that exceptions which occur while calling the :meth:`__getitem__` - method will get suppressed. - To get error reporting use :c:func:`PyObject_GetItem()` instead. + .. versionadded:: 3.13 -.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) +.. c:function:: int PyMapping_HasKeyStringWithError(PyObject *o, const char *key) + + This is the same as :c:func:`PyMapping_HasKeyWithError`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + +.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key) Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. This is equivalent to the Python expression ``key in o``. This function always succeeds. - Note that exceptions which occur while calling the :meth:`__getitem__` - method and creating a temporary string object will get suppressed. - To get error reporting use :c:func:`PyMapping_GetItemString()` instead. + .. note:: + + Exceptions which occur when this calls :meth:`~object.__getitem__` + method are silently ignored. + For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, + :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. + + +.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) + + This is the same as :c:func:`PyMapping_HasKey`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. note:: + + Exceptions that occur when this calls :meth:`~object.__getitem__` + method or while creating the temporary :class:`str` + object are silently ignored. + For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, + :c:func:`PyMapping_GetOptionalItemString` or + :c:func:`PyMapping_GetItemString` instead. .. c:function:: PyObject* PyMapping_Keys(PyObject *o) diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 8e25968c6909fd..489f1580a414b2 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -25,12 +25,16 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. the least-significant 32 bits of *value*; regardless of the size of the native :c:expr:`long` type. *version* indicates the file format. + This function can fail, in which case it sets the error indicator. + Use :c:func:`PyErr_Occurred` to check for that. .. c:function:: void PyMarshal_WriteObjectToFile(PyObject *value, FILE *file, int version) Marshal a Python object, *value*, to *file*. *version* indicates the file format. + This function can fail, in which case it sets the error indicator. + Use :c:func:`PyErr_Occurred` to check for that. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 7041c15d23fb83..1f392e55078e77 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -136,7 +136,7 @@ need to be held. The :ref:`default raw memory allocator ` uses the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` -and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting +and :c:func:`!free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes. .. versionadded:: 3.4 @@ -264,14 +264,14 @@ The following type-oriented macros are provided for convenience. Note that *TYPE* refers to any C type. -.. c:function:: TYPE* PyMem_New(TYPE, size_t n) +.. c:macro:: PyMem_New(TYPE, n) Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of memory. Returns a pointer cast to :c:expr:`TYPE*`. The memory will not have been initialized in any way. -.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) +.. c:macro:: PyMem_Resize(p, TYPE, n) Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * sizeof(TYPE))`` bytes. Returns a pointer cast to :c:expr:`TYPE*`. On return, @@ -391,6 +391,8 @@ Legend: * ``malloc``: system allocators from the standard C library, C functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. * ``pymalloc``: :ref:`pymalloc memory allocator `. +* ``mimalloc``: :ref:`mimalloc memory allocator `. The pymalloc + allocator will be used if mimalloc support isn't available. * "+ debug": with :ref:`debug hooks on the Python memory allocators `. * "Debug build": :ref:`Python build in debug mode `. @@ -423,7 +425,7 @@ Customize Memory Allocators +----------------------------------------------------------+---------------------------------------+ .. versionchanged:: 3.5 - The :c:type:`PyMemAllocator` structure was renamed to + The :c:type:`!PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. @@ -431,6 +433,8 @@ Customize Memory Allocators Enum used to identify an allocator domain. Domains: + .. c:namespace:: NULL + .. c:macro:: PYMEM_DOMAIN_RAW Functions: @@ -470,10 +474,14 @@ Customize Memory Allocators The new allocator must return a distinct non-``NULL`` pointer when requesting zero bytes. - For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be + For the :c:macro:`PYMEM_DOMAIN_RAW` domain, the allocator must be thread-safe: the :term:`GIL ` is not held when the allocator is called. + For the remaining domains, the allocator must also be thread-safe: + the allocator may be called in different interpreters that do not + share a ``GIL``. + If the new allocator is not a hook (does not call the previous allocator), the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the debug hooks on top on the new allocator. @@ -485,19 +493,21 @@ Customize Memory Allocators :c:func:`PyMem_SetAllocator` does have the following contract: - * It can be called after :c:func:`Py_PreInitialize` and before - :c:func:`Py_InitializeFromConfig` to install a custom memory - allocator. There are no restrictions over the installed allocator - other than the ones imposed by the domain (for instance, the Raw - Domain allows the allocator to be called without the GIL held). See - :ref:`the section on allocator domains ` for more - information. + * It can be called after :c:func:`Py_PreInitialize` and before + :c:func:`Py_InitializeFromConfig` to install a custom memory + allocator. There are no restrictions over the installed allocator + other than the ones imposed by the domain (for instance, the Raw + Domain allows the allocator to be called without the GIL held). See + :ref:`the section on allocator domains ` for more + information. - * If called after Python has finish initializing (after - :c:func:`Py_InitializeFromConfig` has been called) the allocator - **must** wrap the existing allocator. Substituting the current - allocator for some other arbitrary one is **not supported**. + * If called after Python has finish initializing (after + :c:func:`Py_InitializeFromConfig` has been called) the allocator + **must** wrap the existing allocator. Substituting the current + allocator for some other arbitrary one is **not supported**. + .. versionchanged:: 3.12 + All allocators must be thread-safe. .. c:function:: void PyMem_SetupDebugHooks(void) @@ -536,8 +546,8 @@ Runtime checks: - Detect write before the start of the buffer (buffer underflow). - Detect write after the end of the buffer (buffer overflow). - Check that the :term:`GIL ` is held when - allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: - :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: + allocator functions of :c:macro:`PYMEM_DOMAIN_OBJ` (ex: + :c:func:`PyObject_Malloc`) and :c:macro:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. On error, the debug hooks use the :mod:`tracemalloc` module to get the @@ -557,9 +567,9 @@ that the treatment of negative indices differs from a Python slice): ``p[-S]`` API identifier (ASCII character): - * ``'r'`` for :c:data:`PYMEM_DOMAIN_RAW`. - * ``'m'`` for :c:data:`PYMEM_DOMAIN_MEM`. - * ``'o'`` for :c:data:`PYMEM_DOMAIN_OBJ`. + * ``'r'`` for :c:macro:`PYMEM_DOMAIN_RAW`. + * ``'m'`` for :c:macro:`PYMEM_DOMAIN_MEM`. + * ``'o'`` for :c:macro:`PYMEM_DOMAIN_OBJ`. ``p[-S+1:0]`` Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads. @@ -581,7 +591,7 @@ that the treatment of negative indices differs from a Python slice): default). A serial number, incremented by 1 on each call to a malloc-like or - realloc-like function. Big-endian ``size_t``. If "bad memory" is detected + realloc-like function. Big-endian :c:type:`size_t`. If "bad memory" is detected later, the serial number gives an excellent way to set a breakpoint on the next run, to capture the instant at which this block was passed out. The static function bumpserialno() in obmalloc.c is the only place the serial @@ -601,7 +611,7 @@ PYMEM_CLEANBYTE (meaning uninitialized memory is getting used). compiled in release mode. On error, the debug hooks now use :mod:`tracemalloc` to get the traceback where a memory block was allocated. The debug hooks now also check if the GIL is held when functions of - :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are + :c:macro:`PYMEM_DOMAIN_OBJ` and :c:macro:`PYMEM_DOMAIN_MEM` domains are called. .. versionchanged:: 3.8 @@ -618,17 +628,18 @@ The pymalloc allocator Python has a *pymalloc* allocator optimized for small objects (smaller or equal to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" -with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and +with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit +platforms. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the :ref:`default allocator ` of the -:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and -:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. +:c:macro:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and +:c:macro:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: -* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, -* :c:func:`mmap` and :c:func:`munmap` if available, +* :c:func:`!VirtualAlloc` and :c:func:`!VirtualFree` on Windows, +* :c:func:`!mmap` and :c:func:`!munmap` if available, * :c:func:`malloc` and :c:func:`free` otherwise. This allocator is disabled if Python is configured with the @@ -663,6 +674,16 @@ Customize pymalloc Arena Allocator Set the arena allocator. +.. _mimalloc: + +The mimalloc allocator +====================== + +.. versionadded:: 3.13 + +Python supports the mimalloc allocator when the underlying platform support is available. +mimalloc "is a general purpose allocator with excellent performance characteristics. +Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages." tracemalloc C API ================= @@ -732,8 +753,8 @@ allocators operating on different heaps. :: free(buf1); /* Fatal -- should be PyMem_Del() */ In addition to the functions aimed at handling raw memory blocks from the Python -heap, objects in Python are allocated and released with :c:func:`PyObject_New`, -:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. +heap, objects in Python are allocated and released with :c:macro:`PyObject_New`, +:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Del`. These will be explained in the next chapter on defining and implementing new object types in C. diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 93ad30cd4f7a8d..0d75ab8e1af111 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -7,8 +7,8 @@ Instance Method Objects .. index:: pair: object; instancemethod -An instance method is a wrapper for a :c:data:`PyCFunction` and the new way -to bind a :c:data:`PyCFunction` to a class object. It replaces the former call +An instance method is a wrapper for a :c:type:`PyCFunction` and the new way +to bind a :c:type:`PyCFunction` to a class object. It replaces the former call ``PyMethod_New(func, NULL, class)``. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 230b471d473be7..979b22261efa3b 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -119,7 +119,7 @@ Module Objects encoded to 'utf-8'. .. deprecated:: 3.2 - :c:func:`PyModule_GetFilename` raises :c:type:`UnicodeEncodeError` on + :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. @@ -145,7 +145,7 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: PyModuleDef_Base m_base - Always initialize this member to :const:`PyModuleDef_HEAD_INIT`. + Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. .. c:member:: const char *m_name @@ -164,7 +164,7 @@ or request "multi-phase initialization" by returning the definition struct itsel This memory area is allocated based on *m_size* on module creation, and freed when the module object is deallocated, after the - :c:member:`m_free` function has been called, if present. + :c:member:`~PyModuleDef.m_free` function has been called, if present. Setting ``m_size`` to ``-1`` means that the module does not support sub-interpreters, because it has global state. @@ -202,7 +202,7 @@ or request "multi-phase initialization" by returning the definition struct itsel This function is not called if the module state was requested but is not allocated yet. This is the case immediately after the module is created and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`m_size` is greater + precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater than 0 and the module state (as returned by :c:func:`PyModule_GetState`) is ``NULL``. @@ -217,7 +217,7 @@ or request "multi-phase initialization" by returning the definition struct itsel This function is not called if the module state was requested but is not allocated yet. This is the case immediately after the module is created and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`m_size` is greater + precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater than 0 and the module state (as returned by :c:func:`PyModule_GetState`) is ``NULL``. @@ -238,7 +238,7 @@ or request "multi-phase initialization" by returning the definition struct itsel This function is not called if the module state was requested but is not allocated yet. This is the case immediately after the module is created and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`m_size` is greater + precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater than 0 and the module state (as returned by :c:func:`PyModule_GetState`) is ``NULL``. @@ -256,7 +256,7 @@ of the following two module creation functions: Create a new module object, given the definition in *def*. This behaves like :c:func:`PyModule_Create2` with *module_api_version* set to - :const:`PYTHON_API_VERSION`. + :c:macro:`PYTHON_API_VERSION`. .. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) @@ -282,7 +282,7 @@ An alternate way to specify extensions is to request "multi-phase initialization Extension modules created this way behave more like Python modules: the initialization is split between the *creation phase*, when the module object is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`__new__` and :py:meth:`__init__` methods +The distinction is similar to the :py:meth:`!__new__` and :py:meth:`!__init__` methods of classes. Unlike modules created using single-phase initialization, these modules are not @@ -293,7 +293,7 @@ By default, multiple modules created from the same definition should be independent: changes to one should not affect the others. This means that all state should be specific to the module object (using e.g. using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`__dict__` or individual classes created with :c:func:`PyType_FromSpec`). +:attr:`~object.__dict__` or individual classes created with :c:func:`PyType_FromSpec`). All modules created using multi-phase initialization are expected to support :ref:`sub-interpreters `. Making sure multiple modules @@ -338,6 +338,7 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) + :noindex: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -372,10 +373,44 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) + :noindex: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. +.. c:macro:: Py_mod_multiple_interpreters + + Specifies one of the following values: + + .. c:namespace:: NULL + + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + + The module does not support being imported in subinterpreters. + + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED + + The module supports being imported in subinterpreters, + but only when they share the main interpreter's GIL. + (See :ref:`isolating-extensions-howto`.) + + .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + + The module supports being imported in subinterpreters, + even when they have their own GIL. + (See :ref:`isolating-extensions-howto`.) + + This slot determines whether or not importing this module + in a subinterpreter will fail. + + Multiple ``Py_mod_multiple_interpreters`` slots may not be specified + in one module definition. + + If ``Py_mod_multiple_interpreters`` is not specified, the import + machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED``. + + .. versionadded:: 3.12 + See :PEP:`489` for more details on multi-phase initialization. Low-level module creation functions @@ -390,7 +425,7 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and Create a new module object, given the definition in *def* and the ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` - with *module_api_version* set to :const:`PYTHON_API_VERSION`. + with *module_api_version* set to :c:macro:`PYTHON_API_VERSION`. .. versionadded:: 3.5 @@ -486,59 +521,56 @@ state: .. versionadded:: 3.10 +.. c:function:: int PyModule_Add(PyObject *module, const char *name, PyObject *value) + + Similar to :c:func:`PyModule_AddObjectRef`, but "steals" a reference + to *value*. + It can be called with a result of function that returns a new reference + without bothering to check its result or even saving it to a variable. + + Example usage:: + + if (PyModule_Add(module, "spam", PyBytes_FromString(value)) < 0) { + goto error; + } + + .. versionadded:: 3.13 + + .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to *value* on success (if it returns ``0``). - The new :c:func:`PyModule_AddObjectRef` function is recommended, since it is + The new :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` + functions are recommended, since it is easy to introduce reference leaks by misusing the :c:func:`PyModule_AddObject` function. .. note:: Unlike other functions that steal references, ``PyModule_AddObject()`` - only decrements the reference count of *value* **on success**. + only releases the reference to *value* **on success**. This means that its return value must be checked, and calling code must - :c:func:`Py_DECREF` *value* manually on error. + :c:func:`Py_XDECREF` *value* manually on error. Example usage:: - static int - add_spam(PyObject *module, int value) - { - PyObject *obj = PyLong_FromLong(value); - if (obj == NULL) { - return -1; - } - if (PyModule_AddObject(module, "spam", obj) < 0) { - Py_DECREF(obj); - return -1; - } - // PyModule_AddObject() stole a reference to obj: - // Py_DECREF(obj) is not needed here - return 0; - } - - The example can also be written without checking explicitly if *obj* is - ``NULL``:: + PyObject *obj = PyBytes_FromString(value); + if (PyModule_AddObject(module, "spam", obj) < 0) { + // If 'obj' is not NULL and PyModule_AddObject() failed, + // 'obj' strong reference must be deleted with Py_XDECREF(). + // If 'obj' is NULL, Py_XDECREF() does nothing. + Py_XDECREF(obj); + goto error; + } + // PyModule_AddObject() stole a reference to obj: + // Py_XDECREF(obj) is not needed here. - static int - add_spam(PyObject *module, int value) - { - PyObject *obj = PyLong_FromLong(value); - if (PyModule_AddObject(module, "spam", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - // PyModule_AddObject() stole a reference to obj: - // Py_DECREF(obj) is not needed here - return 0; - } + .. deprecated:: 3.13 - Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in - this case, since *obj* can be ``NULL``. + :c:func:`PyModule_AddObject` is :term:`soft deprecated`. .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) @@ -555,7 +587,7 @@ state: ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. -.. c:function:: int PyModule_AddIntMacro(PyObject *module, macro) +.. c:macro:: PyModule_AddIntMacro(module, macro) Add an int constant to *module*. The name and the value are taken from *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int @@ -563,7 +595,7 @@ state: Return ``-1`` on error, ``0`` on success. -.. c:function:: int PyModule_AddStringMacro(PyObject *module, macro) +.. c:macro:: PyModule_AddStringMacro(module, macro) Add a string constant to *module*. diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index b84a16a28ead56..f1941fc4bc4e85 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -9,18 +9,18 @@ The ``None`` Object Note that the :c:type:`PyTypeObject` for ``None`` is not directly exposed in the Python/C API. Since ``None`` is a singleton, testing for object identity (using -``==`` in C) is sufficient. There is no :c:func:`PyNone_Check` function for the +``==`` in C) is sufficient. There is no :c:func:`!PyNone_Check` function for the same reason. .. c:var:: PyObject* Py_None - The Python ``None`` object, denoting lack of value. This object has no methods. - It needs to be treated just like any other object with respect to reference - counts. + The Python ``None`` object, denoting lack of value. This object has no methods + and is :term:`immortal`. + .. versionchanged:: 3.12 + :c:data:`Py_None` is :term:`immortal`. .. c:macro:: Py_RETURN_NONE - Properly handle returning :c:data:`Py_None` from within a C function (that is, - increment the reference count of ``None`` and return it.) + Return :c:data:`Py_None` from a function. diff --git a/Doc/c-api/objbuffer.rst b/Doc/c-api/objbuffer.rst deleted file mode 100644 index 6b82a642d7ee42..00000000000000 --- a/Doc/c-api/objbuffer.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. highlight:: c - -Old Buffer Protocol -------------------- - -.. deprecated:: 3.0 - -These functions were part of the "old buffer protocol" API in Python 2. -In Python 3, this protocol doesn't exist anymore but the functions are still -exposed to ease porting 2.x code. They act as a compatibility wrapper -around the :ref:`new buffer protocol `, but they don't give -you control over the lifetime of the resources acquired when a buffer is -exported. - -Therefore, it is recommended that you call :c:func:`PyObject_GetBuffer` -(or the ``y*`` or ``w*`` :ref:`format codes ` with the -:c:func:`PyArg_ParseTuple` family of functions) to get a buffer view over -an object, and :c:func:`PyBuffer_Release` when the buffer view can be released. - - -.. c:function:: int PyObject_AsCharBuffer(PyObject *obj, const char **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a read-only memory location usable as character-based - input. The *obj* argument must support the single-segment character buffer - interface. On success, returns ``0``, sets *buffer* to the memory location - and *buffer_len* to the buffer length. Returns ``-1`` and sets a - :exc:`TypeError` on error. - - -.. c:function:: int PyObject_AsReadBuffer(PyObject *obj, const void **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a read-only memory location containing arbitrary data. - The *obj* argument must support the single-segment readable buffer - interface. On success, returns ``0``, sets *buffer* to the memory location - and *buffer_len* to the buffer length. Returns ``-1`` and sets a - :exc:`TypeError` on error. - - -.. c:function:: int PyObject_CheckReadBuffer(PyObject *o) - - Returns ``1`` if *o* supports the single-segment readable buffer interface. - Otherwise returns ``0``. This function always succeeds. - - Note that this function tries to get and release a buffer, and exceptions - which occur while calling corresponding functions will get suppressed. - To get error reporting use :c:func:`PyObject_GetBuffer()` instead. - - -.. c:function:: int PyObject_AsWriteBuffer(PyObject *obj, void **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a writable memory location. The *obj* argument must - support the single-segment, character buffer interface. On success, - returns ``0``, sets *buffer* to the memory location and *buffer_len* to the - buffer length. Returns ``-1`` and sets a :exc:`TypeError` on error. - diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a25ff244c9f07c..a4e3e74861a315 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -15,39 +15,64 @@ Object Protocol .. c:macro:: Py_RETURN_NOTIMPLEMENTED Properly handle returning :c:data:`Py_NotImplemented` from within a C - function (that is, increment the reference count of NotImplemented and - return it). + function (that is, create a new :term:`strong reference` + to NotImplemented and return it). .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument is used to enable certain printing options. The only option currently supported - is :const:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written + is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written instead of the :func:`repr`. +.. c:function:: int PyObject_HasAttrWithError(PyObject *o, const char *attr_name) + + Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. + This is equivalent to the Python expression ``hasattr(o, attr_name)``. + On failure, return ``-1``. + + .. versionadded:: 3.13 + + +.. c:function:: int PyObject_HasAttrStringWithError(PyObject *o, const char *attr_name) + + This is the same as :c:func:`PyObject_HasAttrWithError`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: int PyObject_HasAttr(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This is equivalent to the Python expression ``hasattr(o, attr_name)``. This function always succeeds. - Note that exceptions which occur while calling :meth:`__getattr__` and - :meth:`__getattribute__` methods will get suppressed. - To get error reporting use :c:func:`PyObject_GetAttr()` instead. + .. note:: + + Exceptions that occur when this calls :meth:`~object.__getattr__` and + :meth:`~object.__getattribute__` methods are silently ignored. + For proper error handling, use :c:func:`PyObject_HasAttrWithError`, + :c:func:`PyObject_GetOptionalAttr` or :c:func:`PyObject_GetAttr` instead. .. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name) - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. + This is the same as :c:func:`PyObject_HasAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. note:: - Note that exceptions which occur while calling :meth:`__getattr__` and - :meth:`__getattribute__` methods and creating a temporary string object - will get suppressed. - To get error reporting use :c:func:`PyObject_GetAttrString()` instead. + Exceptions that occur when this calls :meth:`~object.__getattr__` and + :meth:`~object.__getattribute__` methods or while creating the temporary + :class:`str` object are silently ignored. + For proper error handling, use :c:func:`PyObject_HasAttrStringWithError`, + :c:func:`PyObject_GetOptionalAttrString` + or :c:func:`PyObject_GetAttrString` instead. .. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name) @@ -56,13 +81,42 @@ Object Protocol value on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o.attr_name``. + If the missing attribute should not be treated as a failure, you can use + :c:func:`PyObject_GetOptionalAttr` instead. + .. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name) - Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or ``NULL`` on failure. This is the equivalent of the Python - expression ``o.attr_name``. + This is the same as :c:func:`PyObject_GetAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + If the missing attribute should not be treated as a failure, you can use + :c:func:`PyObject_GetOptionalAttrString` instead. + + +.. c:function:: int PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result); + + Variant of :c:func:`PyObject_GetAttr` which doesn't raise + :exc:`AttributeError` if the attribute is not found. + If the attribute is found, return ``1`` and set *\*result* to a new + :term:`strong reference` to the attribute. + If the attribute is not found, return ``0`` and set *\*result* to ``NULL``; + the :exc:`AttributeError` is silenced. + If an error other than :exc:`AttributeError` is raised, return ``-1`` and + set *\*result* to ``NULL``. + + .. versionadded:: 3.13 + + +.. c:function:: int PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result); + + This is the same as :c:func:`PyObject_GetOptionalAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: 3.13 .. c:function:: PyObject* PyObject_GenericGetAttr(PyObject *o, PyObject *name) @@ -88,10 +142,9 @@ Object Protocol .. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) - Set the value of the attribute named *attr_name*, for object *o*, to the value - *v*. Raise an exception and return ``-1`` on failure; - return ``0`` on success. This is the equivalent of the Python statement - ``o.attr_name = v``. + This is the same as :c:func:`PyObject_SetAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. @@ -117,8 +170,9 @@ Object Protocol .. c:function:: int PyObject_DelAttrString(PyObject *o, const char *attr_name) - Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure. - This is the equivalent of the Python statement ``del o.attr_name``. + This is the same as :c:func:`PyObject_DelAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) @@ -158,8 +212,8 @@ Object Protocol .. c:function:: PyObject* PyObject_RichCompare(PyObject *o1, PyObject *o2, int opid) Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :const:`Py_LT`, :const:`Py_LE`, :const:`Py_EQ`, - :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``, + which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`, + :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``, ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. This is the equivalent of the Python expression ``o1 op o2``, where ``op`` is the operator corresponding to *opid*. Returns the value of the comparison on success, or ``NULL`` on failure. @@ -168,8 +222,8 @@ Object Protocol .. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :const:`Py_LT`, :const:`Py_LE`, :const:`Py_EQ`, - :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``, + which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`, + :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``, ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error, ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the Python expression ``o1 op o2``, where ``op`` is the operator corresponding to @@ -177,7 +231,7 @@ Object Protocol .. note:: If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool` - will always return ``1`` for :const:`Py_EQ` and ``0`` for :const:`Py_NE`. + will always return ``1`` for :c:macro:`Py_EQ` and ``0`` for :c:macro:`Py_NE`. .. c:function:: PyObject* PyObject_Format(PyObject *obj, PyObject *format_spec) @@ -252,7 +306,7 @@ Object Protocol Normally only class objects, i.e. instances of :class:`type` or a derived class, are considered classes. However, objects can override this by having - a :attr:`__bases__` attribute (which must be a tuple of base classes). + a :attr:`~class.__bases__` attribute (which must be a tuple of base classes). .. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) @@ -269,10 +323,10 @@ Object Protocol is an instance of *cls* if its class is a subclass of *cls*. An instance *inst* can override what is considered its class by having a - :attr:`__class__` attribute. + :attr:`~instance.__class__` attribute. An object *cls* can override if it is considered a class, and what its base - classes are, by having a :attr:`__bases__` attribute (which must be a tuple + classes are, by having a :attr:`~class.__bases__` attribute (which must be a tuple of base classes). @@ -316,11 +370,12 @@ Object Protocol When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This - is equivalent to the Python expression ``type(o)``. This function increments the - reference count of the return value. There's really no reason to use this + is equivalent to the Python expression ``type(o)``. + This function creates a new :term:`strong reference` to the return value. + There's really no reason to use this function instead of the :c:func:`Py_TYPE()` function, which returns a - pointer of type :c:expr:`PyTypeObject*`, except when the incremented reference - count is needed. + pointer of type :c:expr:`PyTypeObject*`, except when a new + :term:`strong reference` is needed. .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) @@ -427,10 +482,28 @@ Object Protocol .. c:function:: void *PyObject_GetItemData(PyObject *o) Get a pointer to per-item data for a class with - :const:`Py_TPFLAGS_ITEMS_AT_END`. + :c:macro:`Py_TPFLAGS_ITEMS_AT_END`. On error, set an exception and return ``NULL``. :py:exc:`TypeError` is raised if *o* does not have - :const:`Py_TPFLAGS_ITEMS_AT_END` set. + :c:macro:`Py_TPFLAGS_ITEMS_AT_END` set. .. versionadded:: 3.12 + +.. c:function:: int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) + + Visit the managed dictionary of *obj*. + + This function must only be called in a traverse function of the type which + has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. + + .. versionadded:: 3.13 + +.. c:function:: void PyObject_ClearManagedDict(PyObject *obj) + + Clear the managed dictionary of *obj*. + + This function must only be called in a traverse function of the type which + has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst new file mode 100644 index 00000000000000..3d44d2eb6bf41d --- /dev/null +++ b/Doc/c-api/perfmaps.rst @@ -0,0 +1,50 @@ +.. highlight:: c + +.. _perfmaps: + +Support for Perf Maps +---------------------- + +On supported platforms (as of this writing, only Linux), the runtime can take +advantage of *perf map files* to make Python functions visible to an external +profiling tool (such as `perf `_). +A running process may create a file in the ``/tmp`` directory, which contains entries +that can map a section of executable code to a name. This interface is described in the +`documentation of the Linux Perf tool `_. + +In Python, these helper APIs can be used by libraries and features that rely +on generating machine code on the fly. + +Note that holding the Global Interpreter Lock (GIL) is not required for these APIs. + +.. c:function:: int PyUnstable_PerfMapState_Init(void) + + Open the ``/tmp/perf-$pid.map`` file, unless it's already opened, and create + a lock to ensure thread-safe writes to the file (provided the writes are + done through :c:func:`PyUnstable_WritePerfMapEntry`). Normally, there's no need + to call this explicitly; just use :c:func:`PyUnstable_WritePerfMapEntry` + and it will initialize the state on first call. + + Returns ``0`` on success, ``-1`` on failure to create/open the perf map file, + or ``-2`` on failure to create a lock. Check ``errno`` for more information + about the cause of a failure. + +.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name) + + Write one single entry to the ``/tmp/perf-$pid.map`` file. This function is + thread safe. Here is what an example entry looks like:: + + # address size name + 7f3529fcf759 b py::bar:/run/t.py + + Will call :c:func:`PyUnstable_PerfMapState_Init` before writing the entry, if + the perf map file is not already opened. Returns ``0`` on success, or the + same error codes as :c:func:`PyUnstable_PerfMapState_Init` on failure. + +.. c:function:: void PyUnstable_PerfMapState_Fini(void) + + Close the perf map file opened by :c:func:`PyUnstable_PerfMapState_Init`. + This is called by the runtime itself during interpreter shut-down. In + general, there shouldn't be a reason to explicitly call this, except to + handle specific scenarios such as forking. diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index d8e9c2da6f3ff3..75e1d46474f1e7 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -15,6 +15,12 @@ of Python objects. Get the reference count of the Python object *o*. + Note that the returned value may not actually reflect how many + references to the object are actually held. For example, some + objects are :term:`immortal` and have a very high refcount that does not + reflect the actual number of references. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. + Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. .. versionchanged:: 3.11 @@ -28,36 +34,56 @@ of Python objects. Set the object *o* reference counter to *refcnt*. + On :ref:`Python build with Free Threading `, if + *refcnt* is larger than ``UINT32_MAX``, the object is made :term:`immortal`. + + This function has no effect on :term:`immortal` objects. + .. versionadded:: 3.9 + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. + Indicate taking a new :term:`strong reference` to object *o*, + indicating it is in use and should not be destroyed. + + This function has no effect on :term:`immortal` objects. This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. + When done using the object, release is by calling :c:func:`Py_DECREF`. + The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + Do not expect this function to actually modify *o* in any way. + For at least `some objects `_, + this function has no effect. + + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_XINCREF(PyObject *o) - Increment the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect. + Similar to :c:func:`Py_INCREF`, but the object *o* can be ``NULL``, + in which case this has no effect. See also :c:func:`Py_XNewRef`. .. c:function:: PyObject* Py_NewRef(PyObject *o) - Create a new :term:`strong reference` to an object: increment the reference - count of the object *o* and return the object *o*. + Create a new :term:`strong reference` to an object: + call :c:func:`Py_INCREF` on *o* and return the object *o*. When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` - should be called on it to decrement the object reference count. + should be called on it to release the reference. The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be ``NULL``. @@ -87,9 +113,14 @@ of Python objects. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. + Release a :term:`strong reference` to object *o*, indicating the + reference is no longer used. - If the reference count reaches zero, the object's type's deallocation + This function has no effect on :term:`immortal` objects. + + Once the last :term:`strong reference` is released + (i.e. the object's reference count reaches 0), + the object's type's deallocation function (which must not be ``NULL``) is invoked. This function is usually used to delete a :term:`strong reference` before @@ -98,10 +129,14 @@ of Python objects. The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. + Do not expect this function to actually modify *o* in any way. + For at least `some objects `_, + this function has no effect. + .. warning:: The deallocation function can cause arbitrary Python code to be invoked (e.g. - when a class instance with a :meth:`__del__` method is deallocated). While + when a class instance with a :meth:`~object.__del__` method is deallocated). While exceptions in such code are not propagated, the executed code has free access to all Python global variables. This means that any object that is reachable from a global variable should be in a consistent state before :c:func:`Py_DECREF` is @@ -109,25 +144,29 @@ of Python objects. reference to the deleted object in a temporary variable, update the list data structure, and then call :c:func:`Py_DECREF` for the temporary variable. + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_XDECREF(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, and the same warning applies. + Similar to :c:func:`Py_DECREF`, but the object *o* can be ``NULL``, + in which case this has no effect. + The same warning from :c:func:`Py_DECREF` applies here as well. .. c:function:: void Py_CLEAR(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in + Release a :term:`strong reference` for object *o*. + The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning for :c:func:`Py_DECREF` does not apply with respect to the object passed because the macro carefully uses a temporary variable and sets the argument to ``NULL`` - before decrementing its reference count. + before releasing the reference. - It is a good idea to use this macro whenever decrementing the reference - count of an object that might be traversed during garbage collection. + It is a good idea to use this macro whenever releasing a reference + to an object that might be traversed during garbage collection. .. versionchanged:: 3.12 The macro argument is now only evaluated once. If the argument has side @@ -136,20 +175,22 @@ of Python objects. .. c:function:: void Py_IncRef(PyObject *o) - Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. + Indicate taking a new :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XINCREF`. It can be used for runtime dynamic embedding of Python. .. c:function:: void Py_DecRef(PyObject *o) - Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. + Release a :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XDECREF`. It can be used for runtime dynamic embedding of Python. .. c:macro:: Py_SETREF(dst, src) - Macro safely decrementing the `dst` reference count and setting `dst` to - `src`. + Macro safely releasing a :term:`strong reference` to object *dst* + and setting *dst* to *src*. As in case of :c:func:`Py_CLEAR`, "the obvious" code can be deadly:: @@ -160,9 +201,10 @@ of Python objects. Py_SETREF(dst, src); - That arranges to set `dst` to `src` _before_ decrementing reference count of - *dst* old value, so that any code triggered as a side-effect of `dst` - getting torn down no longer believes `dst` points to a valid object. + That arranges to set *dst* to *src* _before_ releasing the reference + to the old value of *dst*, so that any code triggered as a side-effect + of *dst* getting torn down no longer believes *dst* points + to a valid object. .. versionadded:: 3.6 diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index 402a3e5e09ff56..ce28839f5ba739 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -9,7 +9,7 @@ Sequence Protocol .. c:function:: int PySequence_Check(PyObject *o) Return ``1`` if the object provides the sequence protocol, and ``0`` otherwise. - Note that it returns ``1`` for Python classes with a :meth:`__getitem__` + Note that it returns ``1`` for Python classes with a :meth:`~object.__getitem__` method, unless they are :class:`dict` subclasses, since in general it is impossible to determine what type of keys the class supports. This function always succeeds. diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index d642a5f1902e2e..cba823aa027bd6 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -110,7 +110,7 @@ or :class:`frozenset` or instances of their subtypes. .. index:: pair: built-in function; len Return the length of a :class:`set` or :class:`frozenset` object. Equivalent to - ``len(anyset)``. Raises a :exc:`PyExc_SystemError` if *anyset* is not a + ``len(anyset)``. Raises a :exc:`SystemError` if *anyset* is not a :class:`set`, :class:`frozenset`, or an instance of a subtype. @@ -122,9 +122,9 @@ or :class:`frozenset` or instances of their subtypes. .. c:function:: int PySet_Contains(PyObject *anyset, PyObject *key) Return ``1`` if found, ``0`` if not found, and ``-1`` if an error is encountered. Unlike - the Python :meth:`__contains__` method, this function does not automatically + the Python :meth:`~object.__contains__` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise a :exc:`TypeError` if - the *key* is unhashable. Raise :exc:`PyExc_SystemError` if *anyset* is not a + the *key* is unhashable. Raise :exc:`SystemError` if *anyset* is not a :class:`set`, :class:`frozenset`, or an instance of a subtype. @@ -147,9 +147,9 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` method, this function does not automatically convert unhashable sets into - temporary frozensets. Raise :exc:`PyExc_SystemError` if *set* is not an + temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. @@ -163,4 +163,6 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. .. c:function:: int PySet_Clear(PyObject *set) - Empty an existing set of all elements. + Empty an existing set of all elements. Return ``0`` on + success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of + :class:`set` or its subtype. diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 8271d9acfb645e..27a1757c745d8b 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -34,7 +34,7 @@ Slice Objects *length* as errors. Returns ``0`` on success and ``-1`` on error with no exception set (unless one of - the indices was not :const:`None` and failed to be converted to an integer, + the indices was not ``None`` and failed to be converted to an integer, in which case ``-1`` is returned with an exception set). You probably do not want to use this function. @@ -113,11 +113,13 @@ Slice Objects Ellipsis Object ---------------- +^^^^^^^^^^^^^^^ .. c:var:: PyObject *Py_Ellipsis - The Python ``Ellipsis`` object. This object has no methods. It needs to be - treated just like any other object with respect to reference counts. Like - :c:data:`Py_None` it is a singleton object. + The Python ``Ellipsis`` object. This object has no methods. Like + :c:data:`Py_None`, it is an :term:`immortal` singleton object. + + .. versionchanged:: 3.12 + :c:data:`Py_Ellipsis` is immortal. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 3721fc0697f5cd..63a100a6f26f24 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -18,11 +18,11 @@ way; see :ref:`stable-abi-platform` below). So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, but will need to be compiled separately for 3.9.x and 3.10.x. -There are two tiers of C API with different stability exepectations: +There are two tiers of C API with different stability expectations: -- *Unstable API*, may change in minor versions without a deprecation period. - It is marked by the ``PyUnstable`` prefix in names. -- *Limited API*, is compatible across several minor releases. +- :ref:`Unstable API `, may change in minor versions without + a deprecation period. It is marked by the ``PyUnstable`` prefix in names. +- :ref:`Limited API `, is compatible across several minor releases. When :c:macro:`Py_LIMITED_API` is defined, only this subset is exposed from ``Python.h``. @@ -55,26 +55,26 @@ CPython development and spend extra effort adjusting to changes. Stable Application Binary Interface =================================== +For simplicity, this document talks about *extensions*, but the Limited API +and Stable ABI work the same way for all uses of the API – for example, +embedding Python. + +.. _limited-c-api: + +Limited C API +------------- + Python 3.2 introduced the *Limited API*, a subset of Python's C API. Extensions that only use the Limited API can be compiled once and work with multiple versions of Python. -Contents of the Limited API are :ref:`listed below `. - -To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain compatible across Python 3.x versions. The Stable ABI contains symbols -exposed in the Limited API, but also other ones – for example, functions -necessary to support older versions of the Limited API. - -(For simplicity, this document talks about *extensions*, but the Limited API -and Stable ABI work the same way for all uses of the API – for example, -embedding Python.) +Contents of the Limited API are :ref:`listed below `. .. c:macro:: Py_LIMITED_API Define this macro before including ``Python.h`` to opt in to only use the Limited API, and to select the Limited API version. - Define ``Py_LIMITED_API`` to the value of :c:data:`PY_VERSION_HEX` + Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` corresponding to the lowest Python version your extension supports. The extension will work without recompilation with all Python 3 releases from the specified one onward, and can use Limited API introduced up to that @@ -87,6 +87,19 @@ embedding Python.) You can also define ``Py_LIMITED_API`` to ``3``. This works the same as ``0x03020000`` (Python 3.2, the version that introduced Limited API). + +.. _stable-abi: + +Stable ABI +---------- + +To enable this, Python provides a *Stable ABI*: a set of symbols that will +remain compatible across Python 3.x versions. + +The Stable ABI contains symbols exposed in the :ref:`Limited API +`, but also other ones – for example, functions necessary to +support older versions of the Limited API. + On Windows, extensions that use the Stable ABI should be linked against ``python3.dll`` rather than a version-specific library such as ``python39.dll``. @@ -131,9 +144,9 @@ Limited API Caveats ------------------- Note that compiling with ``Py_LIMITED_API`` is *not* a complete guarantee that -code conforms to the Limited API or the Stable ABI. ``Py_LIMITED_API`` only -covers definitions, but an API also includes other issues, such as expected -semantics. +code conforms to the :ref:`Limited API ` or the :ref:`Stable ABI +`. ``Py_LIMITED_API`` only covers definitions, but an API also +includes other issues, such as expected semantics. One issue that ``Py_LIMITED_API`` does not guard against is calling a function with arguments that are invalid in a lower Python version. @@ -166,9 +179,9 @@ Platform Considerations ======================= ABI stability depends not only on Python, but also on the compiler used, -lower-level libraries and compiler options. For the purposes of the Stable ABI, -these details define a “platformâ€. They usually depend on the OS -type and processor architecture +lower-level libraries and compiler options. For the purposes of +the :ref:`Stable ABI `, these details define a “platformâ€. They +usually depend on the OS type and processor architecture It is the responsibility of each particular distributor of Python to ensure that all Python versions on a particular platform are built @@ -177,12 +190,12 @@ This is the case with Windows and macOS releases from ``python.org`` and many third-party distributors. -.. _stable-abi-list: +.. _limited-api-list: Contents of Limited API ======================= -Currently, the Limited API includes the following items: +Currently, the :ref:`Limited API ` includes the following items: .. limited-api-list:: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index aae1b951804491..528813c255c0a5 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -35,7 +35,7 @@ under :ref:`reference counting `. .. c:type:: PyVarObject - This is an extension of :c:type:`PyObject` that adds the :attr:`ob_size` + This is an extension of :c:type:`PyObject` that adds the :c:member:`~PyVarObject.ob_size` field. This is only used for objects that have some notion of *length*. This type does not often appear in the Python/C API. Access to the members must be done by using the macros @@ -152,7 +152,7 @@ under :ref:`reference counting `. .. c:macro:: PyVarObject_HEAD_INIT(type, size) This is a macro which expands to initialization values for a new - :c:type:`PyVarObject` type, including the :attr:`ob_size` field. + :c:type:`PyVarObject` type, including the :c:member:`~PyVarObject.ob_size` field. This macro expands to:: _PyObject_EXTRA_INIT @@ -179,7 +179,7 @@ Implementing functions and methods .. c:type:: PyCFunctionWithKeywords Type of the functions used to implement Python callables in C - with signature :const:`METH_VARARGS | METH_KEYWORDS`. + with signature :ref:`METH_VARARGS | METH_KEYWORDS `. The function signature is:: PyObject *PyCFunctionWithKeywords(PyObject *self, @@ -190,7 +190,7 @@ Implementing functions and methods .. c:type:: _PyCFunctionFast Type of the functions used to implement Python callables in C - with signature :const:`METH_FASTCALL`. + with signature :c:macro:`METH_FASTCALL`. The function signature is:: PyObject *_PyCFunctionFast(PyObject *self, @@ -200,7 +200,7 @@ Implementing functions and methods .. c:type:: _PyCFunctionFastWithKeywords Type of the functions used to implement Python callables in C - with signature :const:`METH_FASTCALL | METH_KEYWORDS`. + with signature :ref:`METH_FASTCALL | METH_KEYWORDS `. The function signature is:: PyObject *_PyCFunctionFastWithKeywords(PyObject *self, @@ -211,7 +211,7 @@ Implementing functions and methods .. c:type:: PyCMethod Type of the functions used to implement Python callables in C - with signature :const:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS`. + with signature :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `. The function signature is:: PyObject *PyCMethod(PyObject *self, @@ -228,36 +228,38 @@ Implementing functions and methods Structure used to describe a method of an extension type. This structure has four fields: - .. c:member:: const char* ml_name + .. c:member:: const char *ml_name - name of the method + Name of the method. .. c:member:: PyCFunction ml_meth - pointer to the C implementation + Pointer to the C implementation. .. c:member:: int ml_flags - flags bits indicating how the call should be constructed + Flags bits indicating how the call should be constructed. - .. c:member:: const char* ml_doc + .. c:member:: const char *ml_doc - points to the contents of the docstring + Points to the contents of the docstring. -The :c:member:`ml_meth` is a C function pointer. The functions may be of different +The :c:member:`~PyMethodDef.ml_meth` is a C function pointer. +The functions may be of different types, but they always return :c:expr:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as :c:expr:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. -The :c:member:`ml_flags` field is a bitfield which can include the following flags. +The :c:member:`~PyMethodDef.ml_flags` field is a bitfield which can include +the following flags. The individual flags indicate either a calling convention or a binding convention. There are these calling conventions: -.. data:: METH_VARARGS +.. c:macro:: METH_VARARGS This is the typical calling convention, where the methods have the type :c:type:`PyCFunction`. The function expects two :c:expr:`PyObject*` values. @@ -267,8 +269,17 @@ There are these calling conventions: using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. -.. data:: METH_VARARGS | METH_KEYWORDS +.. c:macro:: METH_KEYWORDS + Can only be used in certain combinations with other flags: + :ref:`METH_VARARGS | METH_KEYWORDS `, + :ref:`METH_FASTCALL | METH_KEYWORDS ` and + :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `. + + +.. _METH_VARARGS-METH_KEYWORDS: + +:c:expr:`METH_VARARGS | METH_KEYWORDS` Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. The function expects three parameters: *self*, *args*, *kwargs* where *kwargs* is a dictionary of all the keyword arguments or possibly ``NULL`` @@ -276,7 +287,7 @@ There are these calling conventions: using :c:func:`PyArg_ParseTupleAndKeywords`. -.. data:: METH_FASTCALL +.. c:macro:: METH_FASTCALL Fast calling convention supporting only positional arguments. The methods have the type :c:type:`_PyCFunctionFast`. @@ -288,12 +299,13 @@ There are these calling conventions: .. versionchanged:: 3.10 - ``METH_FASTCALL`` is now part of the stable ABI. + ``METH_FASTCALL`` is now part of the :ref:`stable ABI `. -.. data:: METH_FASTCALL | METH_KEYWORDS +.. _METH_FASTCALL-METH_KEYWORDS: - Extension of :const:`METH_FASTCALL` supporting also keyword arguments, +:c:expr:`METH_FASTCALL | METH_KEYWORDS` + Extension of :c:macro:`METH_FASTCALL` supporting also keyword arguments, with methods of type :c:type:`_PyCFunctionFastWithKeywords`. Keyword arguments are passed the same way as in the :ref:`vectorcall protocol `: @@ -306,10 +318,18 @@ There are these calling conventions: .. versionadded:: 3.7 -.. data:: METH_METHOD | METH_FASTCALL | METH_KEYWORDS +.. c:macro:: METH_METHOD + + Can only be used in the combination with other flags: + :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `. + - Extension of :const:`METH_FASTCALL | METH_KEYWORDS` supporting the *defining - class*, that is, the class that contains the method in question. +.. _METH_METHOD-METH_FASTCALL-METH_KEYWORDS: + +:c:expr:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS` + Extension of :ref:`METH_FASTCALL | METH_KEYWORDS ` + supporting the *defining class*, that is, + the class that contains the method in question. The defining class might be a superclass of ``Py_TYPE(self)``. The method needs to be of type :c:type:`PyCMethod`, the same as for @@ -319,10 +339,10 @@ There are these calling conventions: .. versionadded:: 3.9 -.. data:: METH_NOARGS +.. c:macro:: METH_NOARGS Methods without parameters don't need to check whether arguments are given if - they are listed with the :const:`METH_NOARGS` flag. They need to be of type + they are listed with the :c:macro:`METH_NOARGS` flag. They need to be of type :c:type:`PyCFunction`. The first parameter is typically named *self* and will hold a reference to the module or object instance. In all cases the second parameter will be ``NULL``. @@ -331,9 +351,9 @@ There are these calling conventions: :c:macro:`Py_UNUSED` can be used to prevent a compiler warning. -.. data:: METH_O +.. c:macro:: METH_O - Methods with a single object argument can be listed with the :const:`METH_O` + Methods with a single object argument can be listed with the :c:macro:`METH_O` flag, instead of invoking :c:func:`PyArg_ParseTuple` with a ``"O"`` argument. They have the type :c:type:`PyCFunction`, with the *self* parameter, and a :c:expr:`PyObject*` parameter representing the single argument. @@ -345,7 +365,7 @@ defined for modules. At most one of these flags may be set for any given method. -.. data:: METH_CLASS +.. c:macro:: METH_CLASS .. index:: pair: built-in function; classmethod @@ -355,7 +375,7 @@ method. function. -.. data:: METH_STATIC +.. c:macro:: METH_STATIC .. index:: pair: built-in function; staticmethod @@ -367,13 +387,13 @@ One other constant controls whether a method is loaded in place of another definition with the same method name. -.. data:: METH_COEXIST +.. c:macro:: METH_COEXIST The method will be loaded in place of existing definitions. Without *METH_COEXIST*, the default is to skip repeated definitions. Since slot wrappers are loaded before the method table, the existence of a *sq_contains* slot, for example, would generate a wrapped method named - :meth:`__contains__` and preclude the loading of a corresponding + :meth:`~object.__contains__` and preclude the loading of a corresponding PyCFunction with the same name. With the flag defined, the PyCFunction will be loaded in place of the wrapper object and will co-exist with the slot. This is helpful because calls to PyCFunctions are optimized more @@ -386,7 +406,11 @@ Accessing attributes of extension types .. c:type:: PyMemberDef Structure which describes an attribute of a type which corresponds to a C - struct member. Its fields are, in order: + struct member. + When defining a class, put a NULL-terminated array of these + structures in the :c:member:`~PyTypeObject.tp_members` slot. + + Its fields are, in order: .. c:member:: const char* name @@ -395,7 +419,7 @@ Accessing attributes of extension types The string should be static, no copy is made of it. - .. c:member:: Py_ssize_t PyMemberDef.offset + .. c:member:: Py_ssize_t offset The offset in bytes that the member is located on the type’s object struct. @@ -414,7 +438,7 @@ Accessing attributes of extension types The string should be static, no copy is made of it. Typically, it is defined using :c:macro:`PyDoc_STR`. - By default (when :c:member:`flags` is ``0``), members allow + By default (when :c:member:`~PyMemberDef.flags` is ``0``), members allow both read and write access. Use the :c:macro:`Py_READONLY` flag for read-only access. Certain types, like :c:macro:`Py_T_STRING`, imply :c:macro:`Py_READONLY`. @@ -440,8 +464,8 @@ Accessing attributes of extension types The legacy offsets :c:member:`~PyTypeObject.tp_dictoffset` and :c:member:`~PyTypeObject.tp_weaklistoffset` can be defined similarly using ``"__dictoffset__"`` and ``"__weaklistoffset__"`` members, but extensions - are strongly encouraged to use :const:`Py_TPFLAGS_MANAGED_DICT` and - :const:`Py_TPFLAGS_MANAGED_WEAKREF` instead. + are strongly encouraged to use :c:macro:`Py_TPFLAGS_MANAGED_DICT` and + :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead. .. versionchanged:: 3.12 @@ -494,7 +518,7 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: Can only be used as part of :c:member:`Py_tp_members ` :c:type:`slot ` when creating a class using negative - :c:member:`~PyTypeDef.basicsize`. + :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. This flag is only used in :c:type:`PyTypeSlot`. @@ -509,19 +533,19 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: .. versionchanged:: 3.10 - The :const:`!RESTRICTED`, :const:`!READ_RESTRICTED` and - :const:`!WRITE_RESTRICTED` macros available with + The :c:macro:`!RESTRICTED`, :c:macro:`!READ_RESTRICTED` and + :c:macro:`!WRITE_RESTRICTED` macros available with ``#include "structmember.h"`` are deprecated. - :const:`!READ_RESTRICTED` and :const:`!RESTRICTED` are equivalent to - :const:`Py_AUDIT_READ`; :const:`!WRITE_RESTRICTED` does nothing. + :c:macro:`!READ_RESTRICTED` and :c:macro:`!RESTRICTED` are equivalent to + :c:macro:`Py_AUDIT_READ`; :c:macro:`!WRITE_RESTRICTED` does nothing. .. index:: single: READONLY .. versionchanged:: 3.12 - The :const:`!READONLY` macro was renamed to :const:`Py_READONLY`. - The :const:`!PY_AUDIT_READ` macro was renamed with the ``Py_`` prefix. + The :c:macro:`!READONLY` macro was renamed to :c:macro:`Py_READONLY`. + The :c:macro:`!PY_AUDIT_READ` macro was renamed with the ``Py_`` prefix. The new names are now always available. Previously, these required ``#include "structmember.h"``. The header is still available and it provides the old names. @@ -568,7 +592,7 @@ Macro name C type Python type (*): Zero-terminated, UTF8-encoded C string. With :c:macro:`!Py_T_STRING` the C representation is a pointer; - with :c:macro:`!Py_T_STRING_INLINE` the string is stored directly + with :c:macro:`!Py_T_STRING_INPLACE` the string is stored directly in the structure. (**): String of length 1. Only ASCII is accepted. @@ -625,23 +649,23 @@ Defining Getters and Setters Structure to define property-like access for a type. See also description of the :c:member:`PyTypeObject.tp_getset` slot. - .. c:member:: const char* PyGetSetDef.name + .. c:member:: const char* name attribute name - .. c:member:: getter PyGetSetDef.get + .. c:member:: getter get C function to get the attribute. - .. c:member:: setter PyGetSetDef.set + .. c:member:: setter set Optional C function to set or delete the attribute, if omitted the attribute is readonly. - .. c:member:: const char* PyGetSetDef.doc + .. c:member:: const char* doc optional docstring - .. c:member:: void* PyGetSetDef.closure + .. c:member:: void* closure Optional function pointer, providing additional data for getter and setter. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 6fc8a3aff95686..e3c54b075114ff 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -8,8 +8,9 @@ Operating System Utilities .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a - :class:`str` or :class:`bytes` object, then its reference count is - incremented. If the object implements the :class:`os.PathLike` interface, + :class:`str` or :class:`bytes` object, then a new + :term:`strong reference` is returned. + If the object implements the :class:`os.PathLike` interface, then :meth:`~os.PathLike.__fspath__` is returned as long as it is a :class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised and ``NULL`` is returned. @@ -97,16 +98,16 @@ Operating System Utilities .. c:function:: int PyOS_CheckStack() Return true when the interpreter runs out of stack space. This is a reliable - check, but is only available when :const:`USE_STACKCHECK` is defined (currently + check, but is only available when :c:macro:`USE_STACKCHECK` is defined (currently on certain versions of Windows using the Microsoft Visual C++ compiler). - :const:`USE_STACKCHECK` will be defined automatically; you should never + :c:macro:`USE_STACKCHECK` will be defined automatically; you should never change the definition in your own code. .. c:function:: PyOS_sighandler_t PyOS_getsig(int i) Return the current signal handler for signal *i*. This is a thin wrapper around - either :c:func:`sigaction` or :c:func:`signal`. Do not call those functions + either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void (\*)(int)`. @@ -114,7 +115,7 @@ Operating System Utilities .. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) Set the signal handler for signal *i* to be *h*; return the old signal handler. - This is a thin wrapper around either :c:func:`sigaction` or :c:func:`signal`. Do + This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void (\*)(int)`. @@ -167,7 +168,7 @@ Operating System Utilities .. versionchanged:: 3.8 The function now uses the UTF-8 encoding on Windows if - :c:member:`PyConfig.legacy_windows_fs_encoding` is zero; + :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero; .. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) @@ -209,7 +210,7 @@ Operating System Utilities .. versionchanged:: 3.8 The function now uses the UTF-8 encoding on Windows if - :c:member:`PyConfig.legacy_windows_fs_encoding` is zero. + :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero. .. _systemfunctions: @@ -237,45 +238,8 @@ accessible to C code. They all work with the current interpreter thread's Reset :data:`sys.warnoptions` to an empty list. This function may be called prior to :c:func:`Py_Initialize`. -.. c:function:: void PySys_AddWarnOption(const wchar_t *s) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.warnoptions` should be used instead, see :ref:`Python - Initialization Configuration `. - - Append *s* to :data:`sys.warnoptions`. This function must be called prior - to :c:func:`Py_Initialize` in order to affect the warnings filter list. - - .. deprecated:: 3.11 - -.. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.warnoptions` should be used instead, see :ref:`Python - Initialization Configuration `. - - Append *unicode* to :data:`sys.warnoptions`. - - Note: this function is not currently usable from outside the CPython - implementation, as it must be called prior to the implicit import of - :mod:`warnings` in :c:func:`Py_Initialize` to be effective, but can't be - called until enough of the runtime has been initialized to permit the - creation of Unicode objects. - - .. deprecated:: 3.11 - -.. c:function:: void PySys_SetPath(const wchar_t *path) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.module_search_paths` and - :c:member:`PyConfig.module_search_paths_set` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set :data:`sys.path` to a list object of paths found in *path* which should - be a list of paths separated with the platform's search path delimiter - (``:`` on Unix, ``;`` on Windows). - - .. deprecated:: 3.11 + .. deprecated-removed:: 3.13 3.15 + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. .. c:function:: void PySys_WriteStdout(const char *format, ...) @@ -313,20 +277,6 @@ accessible to C code. They all work with the current interpreter thread's .. versionadded:: 3.2 -.. c:function:: void PySys_AddXOption(const wchar_t *s) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.xoptions` should be used instead, see :ref:`Python - Initialization Configuration `. - - Parse *s* as a set of :option:`-X` options and add them to the current - options mapping as returned by :c:func:`PySys_GetXOptions`. This function - may be called prior to :c:func:`Py_Initialize`. - - .. versionadded:: 3.2 - - .. deprecated:: 3.11 - .. c:function:: PyObject *PySys_GetXOptions() Return the current dictionary of :option:`-X` options, similarly to @@ -341,19 +291,24 @@ accessible to C code. They all work with the current interpreter thread's Raise an auditing event with any active hooks. Return zero for success and non-zero with an exception set on failure. + The *event* string argument must not be *NULL*. + If any hooks have been added, *format* and other arguments will be used to construct a tuple to pass. Apart from ``N``, the same format characters as used in :c:func:`Py_BuildValue` are available. If the built value is not - a tuple, it will be added into a single-element tuple. (The ``N`` format - option consumes a reference, but since there is no way to know whether - arguments to this function will be consumed, using it may cause reference - leaks.) + a tuple, it will be added into a single-element tuple. + + The ``N`` format option must not be used. It consumes a reference, but since + there is no way to know whether arguments to this function will be consumed, + using it may cause reference leaks. Note that ``#`` format characters should always be treated as :c:type:`Py_ssize_t`, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined. :func:`sys.audit` performs the same function from Python code. + See also :c:func:`PySys_AuditTuple`. + .. versionadded:: 3.8 .. versionchanged:: 3.8.2 @@ -362,6 +317,14 @@ accessible to C code. They all work with the current interpreter thread's unavoidable deprecation warning was raised. +.. c:function:: int PySys_AuditTuple(const char *event, PyObject *args) + + Similar to :c:func:`PySys_Audit`, but pass arguments as a Python object. + *args* must be a :class:`tuple`. To pass no arguments, *args* can be *NULL*. + + .. versionadded:: 3.13 + + .. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) Append the callable *hook* to the list of active auditing hooks. @@ -414,7 +377,7 @@ Process Control This function should only be invoked when a condition is detected that would make it dangerous to continue using the Python interpreter; e.g., when the object administration appears to be corrupted. On Unix, the standard C library - function :c:func:`abort` is called which will attempt to produce a :file:`core` + function :c:func:`!abort` is called which will attempt to produce a :file:`core` file. The ``Py_FatalError()`` function is replaced with a macro which logs diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index ac62058676eeeb..b3710560ebe7ac 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -89,6 +89,9 @@ Tuple Objects Like :c:func:`PyTuple_SetItem`, but does no error checking, and should *only* be used to fill in brand new tuples. + Bounds checking is performed as an assertion if Python is built in + :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. + .. note:: This function "steals" a reference to *o*, and, unlike @@ -111,6 +114,8 @@ Tuple Objects raises :exc:`MemoryError` or :exc:`SystemError`. +.. _struct-sequence-objects: + Struct Sequence Objects ----------------------- @@ -142,39 +147,39 @@ type. Contains the meta information of a struct sequence type to create. - +-------------------+------------------------------+--------------------------------------+ - | Field | C Type | Meaning | - +===================+==============================+======================================+ - | ``name`` | ``const char *`` | name of the struct sequence type | - +-------------------+------------------------------+--------------------------------------+ - | ``doc`` | ``const char *`` | pointer to docstring for the type | - | | | or ``NULL`` to omit | - +-------------------+------------------------------+--------------------------------------+ - | ``fields`` | ``PyStructSequence_Field *`` | pointer to ``NULL``-terminated array | - | | | with field names of the new type | - +-------------------+------------------------------+--------------------------------------+ - | ``n_in_sequence`` | ``int`` | number of fields visible to the | - | | | Python side (if used as tuple) | - +-------------------+------------------------------+--------------------------------------+ + .. c:member:: const char *name + + Name of the struct sequence type. + + .. c:member:: const char *doc + + Pointer to docstring for the type or ``NULL`` to omit. + + .. c:member:: PyStructSequence_Field *fields + + Pointer to ``NULL``-terminated array with field names of the new type. + + .. c:member:: int n_in_sequence + + Number of fields visible to the Python side (if used as tuple). .. c:type:: PyStructSequence_Field Describes a field of a struct sequence. As a struct sequence is modeled as a tuple, all fields are typed as :c:expr:`PyObject*`. The index in the - :attr:`fields` array of the :c:type:`PyStructSequence_Desc` determines which + :c:member:`~PyStructSequence_Desc.fields` array of + the :c:type:`PyStructSequence_Desc` determines which field of the struct sequence is described. - +-----------+------------------+-----------------------------------------+ - | Field | C Type | Meaning | - +===========+==================+=========================================+ - | ``name`` | ``const char *`` | name for the field or ``NULL`` to end | - | | | the list of named fields, set to | - | | | :c:data:`PyStructSequence_UnnamedField` | - | | | to leave unnamed | - +-----------+------------------+-----------------------------------------+ - | ``doc`` | ``const char *`` | field docstring or ``NULL`` to omit | - +-----------+------------------+-----------------------------------------+ + .. c:member:: const char *name + + Name for the field or ``NULL`` to end the list of named fields, + set to :c:data:`PyStructSequence_UnnamedField` to leave unnamed. + + .. c:member:: const char *doc + + Field docstring or ``NULL`` to omit. .. c:var:: const char * const PyStructSequence_UnnamedField @@ -194,12 +199,17 @@ type. .. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) Return the object at position *pos* in the struct sequence pointed to by *p*. - No bounds checking is performed. + + Bounds checking is performed as an assertion if Python is built in + :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. .. c:function:: PyObject* PyStructSequence_GET_ITEM(PyObject *p, Py_ssize_t pos) - Macro equivalent of :c:func:`PyStructSequence_GetItem`. + Alias to :c:func:`PyStructSequence_GetItem`. + + .. versionchanged:: 3.13 + Now implemented as an alias to :c:func:`PyStructSequence_GetItem`. .. c:function:: void PyStructSequence_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) @@ -208,6 +218,9 @@ type. :c:func:`PyTuple_SET_ITEM`, this should only be used to fill in brand new instances. + Bounds checking is performed as an assertion if Python is built in + :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. + .. note:: This function "steals" a reference to *o*. @@ -215,9 +228,7 @@ type. .. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) - Similar to :c:func:`PyStructSequence_SetItem`, but implemented as a static - inlined function. - - .. note:: + Alias to :c:func:`PyStructSequence_SetItem`. - This function "steals" a reference to *o*. + .. versionchanged:: 3.13 + Now implemented as an alias to :c:func:`PyStructSequence_SetItem`. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index fb38935e003336..5aaa8147dd3176 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -42,7 +42,7 @@ Type Objects Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily meant for use with ``Py_LIMITED_API``; the individual flag bits are guaranteed to be stable across Python releases, but access to - :c:member:`~PyTypeObject.tp_flags` itself is not part of the limited API. + :c:member:`~PyTypeObject.tp_flags` itself is not part of the :ref:`limited API `. .. versionadded:: 3.2 @@ -50,6 +50,23 @@ Type Objects The return type is now ``unsigned long`` rather than ``long``. +.. c:function:: PyObject* PyType_GetDict(PyTypeObject* type) + + Return the type object's internal namespace, which is otherwise only + exposed via a read-only proxy (``cls.__dict__``). This is a + replacement for accessing :c:member:`~PyTypeObject.tp_dict` directly. + The returned dictionary must be treated as read-only. + + This function is meant for specific embedding and language-binding cases, + where direct access to the dict is necessary and indirect access + (e.g. via the proxy or :c:func:`PyObject_GetAttr`) isn't adequate. + + Extension modules should continue to use ``tp_dict``, + directly or indirectly, when setting up their own types. + + .. versionadded:: 3.12 + + .. c:function:: void PyType_Modified(PyTypeObject *type) Invalidate the internal lookup cache for the type and all of its @@ -86,7 +103,7 @@ Type Objects :c:func:`PyType_AddWatcher` will be called whenever :c:func:`PyType_Modified` reports a change to *type*. (The callback may be called only once for a series of consecutive modifications to *type*, if - :c:func:`PyType_Lookup` is not called on *type* between the modifications; + :c:func:`!_PyType_Lookup` is not called on *type* between the modifications; this is an implementation detail and subject to change.) An extension should never call ``PyType_Watch`` with a *watcher_id* that was @@ -115,7 +132,7 @@ Type Objects .. c:function:: int PyType_IS_GC(PyTypeObject *o) Return true if the type object includes support for the cycle detector; this - tests the type flag :const:`Py_TPFLAGS_HAVE_GC`. + tests the type flag :c:macro:`Py_TPFLAGS_HAVE_GC`. .. c:function:: int PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) @@ -148,10 +165,10 @@ Type Objects .. note:: If some of the base classes implements the GC protocol and the provided - type does not include the :const:`Py_TPFLAGS_HAVE_GC` in its flags, then + type does not include the :c:macro:`Py_TPFLAGS_HAVE_GC` in its flags, then the GC protocol will be automatically implemented from its parents. On the contrary, if the type being created does include - :const:`Py_TPFLAGS_HAVE_GC` in its flags then it **must** implement the + :c:macro:`Py_TPFLAGS_HAVE_GC` in its flags then it **must** implement the GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. @@ -198,7 +215,7 @@ Type Objects ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses are not necessarily defined in the same module as their superclass. See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when ``PyCMethod`` cannot + See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot be used. .. versionadded:: 3.9 @@ -251,14 +268,14 @@ The following functions and structs are used to create .. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) Create and return a :ref:`heap type ` from the *spec* - (see :const:`Py_TPFLAGS_HEAPTYPE`). + (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). The metaclass *metaclass* is used to construct the resulting type object. When *metaclass* is ``NULL``, the metaclass is derived from *bases* (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not - supported. + supported, except if ``tp_new`` is ``NULL``. (For backwards compatibility, other ``PyType_From*`` functions allow such metaclasses. They ignore ``tp_new``, which may result in incomplete initialization. This is deprecated and in Python 3.14+ such metaclasses will @@ -349,6 +366,15 @@ The following functions and structs are used to create :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it will be no longer allowed. +.. raw:: html + + + + + + + + .. c:type:: PyType_Spec Structure defining a type's behavior. @@ -394,7 +420,7 @@ The following functions and structs are used to create - The requested :c:member:`PyType_Spec.basicsize` is zero, suggesting that the subclass does not access the instance's memory directly. - - With the :const:`Py_TPFLAGS_ITEMS_AT_END` flag. + - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. .. c:member:: unsigned int flags @@ -410,12 +436,18 @@ The following functions and structs are used to create Each slot ID should be specified at most once. +.. raw:: html + + + + + .. c:type:: PyType_Slot Structure defining optional functionality of a type, containing a slot ID and a value pointer. - .. c:member:: int PyType_Slot.slot + .. c:member:: int slot A slot ID. @@ -429,26 +461,39 @@ The following functions and structs are used to create * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` - The following fields cannot be set at all using :c:type:`PyType_Spec` and - :c:type:`PyType_Slot`: + The following “offset†fields cannot be set using :c:type:`PyType_Slot`: + + * :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) + * :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) + * :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in + :ref:`PyMemberDef `) + + If it is not possible to switch to a ``MANAGED`` flag (for example, + for vectorcall or to support Python older than 3.12), specify the + offset in :c:member:`Py_tp_members `. + See :ref:`PyMemberDef documentation ` + for details. + + The following fields cannot be set at all when creating a heap type: - * :c:member:`~PyTypeObject.tp_dict` - * :c:member:`~PyTypeObject.tp_mro` - * :c:member:`~PyTypeObject.tp_cache` - * :c:member:`~PyTypeObject.tp_subclasses` - * :c:member:`~PyTypeObject.tp_weaklist` * :c:member:`~PyTypeObject.tp_vectorcall` - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :const:`Py_TPFLAGS_MANAGED_WEAKREF` instead) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :const:`Py_TPFLAGS_MANAGED_DICT` instead) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (see :ref:`PyMemberDef `) + (use :c:member:`~PyTypeObject.tp_new` and/or + :c:member:`~PyTypeObject.tp_init`) + + * Internal fields: + :c:member:`~PyTypeObject.tp_dict`, + :c:member:`~PyTypeObject.tp_mro`, + :c:member:`~PyTypeObject.tp_cache`, + :c:member:`~PyTypeObject.tp_subclasses`, and + :c:member:`~PyTypeObject.tp_weaklist`. Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be problematic on some platforms. To avoid issues, use the *bases* argument of - :py:func:`PyType_FromSpecWithBases` instead. + :c:func:`PyType_FromSpecWithBases` instead. .. versionchanged:: 3.9 @@ -457,9 +502,9 @@ The following functions and structs are used to create .. versionchanged:: 3.11 :c:member:`~PyBufferProcs.bf_getbuffer` and :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the limited API. + under the :ref:`limited API `. - .. c:member:: void *PyType_Slot.pfunc + .. c:member:: void *pfunc The desired value of the slot. In most cases, this is a pointer to a function. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 4c1957a2a1dbca..98fe68737deb81 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -35,7 +35,7 @@ two types exist -- :ref:`GenericAlias ` and ... } - .. seealso:: The data model method :meth:`__class_getitem__`. + .. seealso:: The data model method :meth:`~object.__class_getitem__`. .. versionadded:: 3.9 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 0584989233de3f..8a26f237652d12 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -147,7 +147,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_watched`] | char | | | | | | + | [:c:member:`~PyTypeObject.tp_watched`] | unsigned char | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] @@ -163,9 +163,9 @@ Quick Reference .. [#cols] Columns: - **"O"**: set on :c:type:`PyBaseObject_Type` + **"O"**: set on :c:data:`PyBaseObject_Type` - **"T"**: set on :c:type:`PyType_Type` + **"T"**: set on :c:data:`PyType_Type` **"D"**: default (if slot is set to ``NULL``) @@ -343,13 +343,13 @@ slot typedefs | | :c:type:`PyTypeObject` * | | | | :c:type:`Py_ssize_t` | | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`destructor` | void * | void | +| :c:type:`destructor` | :c:type:`PyObject` * | void | +-----------------------------+-----------------------------+----------------------+ | :c:type:`freefunc` | void * | void | +-----------------------------+-----------------------------+----------------------+ | :c:type:`traverseproc` | .. line-block:: | int | | | | | -| | void * | | +| | :c:type:`PyObject` * | | | | :c:type:`visitproc` | | | | void * | | +-----------------------------+-----------------------------+----------------------+ @@ -426,7 +426,7 @@ slot typedefs | | :c:type:`PyObject` * | | | | :c:type:`Py_buffer` * | | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`inquiry` | void * | int | +| :c:type:`inquiry` | :c:type:`PyObject` * | int | +-----------------------------+-----------------------------+----------------------+ | :c:type:`unaryfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | @@ -485,17 +485,17 @@ PyObject Slots -------------- The type object structure extends the :c:type:`PyVarObject` structure. The -:attr:`ob_size` field is used for dynamic types (created by :func:`type_new`, +:c:member:`~PyVarObject.ob_size` field is used for dynamic types (created by :c:func:`!type_new`, usually called from a class statement). Note that :c:data:`PyType_Type` (the metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that its instances (i.e. -type objects) *must* have the :attr:`ob_size` field. +type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. .. c:member:: Py_ssize_t PyObject.ob_refcnt This is the type object's reference count, initialized to ``1`` by the ``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type - objects `, the type's instances (objects whose :attr:`ob_type` + objects `, the type's instances (objects whose :c:member:`~PyObject.ob_type` points back to the type) do *not* count as references. But for :ref:`dynamically allocated type objects `, the instances *do* count as references. @@ -519,8 +519,8 @@ type objects) *must* have the :attr:`ob_size` field. Foo_Type.ob_type = &PyType_Type; This should be done before any instances of the type are created. - :c:func:`PyType_Ready` checks if :attr:`ob_type` is ``NULL``, and if so, - initializes it to the :attr:`ob_type` field of the base class. + :c:func:`PyType_Ready` checks if :c:member:`~PyObject.ob_type` is ``NULL``, and if so, + initializes it to the :c:member:`~PyObject.ob_type` field of the base class. :c:func:`PyType_Ready` will not change this field if it is non-zero. **Inheritance:** @@ -528,28 +528,6 @@ type objects) *must* have the :attr:`ob_size` field. This field is inherited by subtypes. -.. c:member:: PyObject* PyObject._ob_next - PyObject* PyObject._ob_prev - - These fields are only present when the macro ``Py_TRACE_REFS`` is defined - (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). - - Their initialization to ``NULL`` is taken care of by the - ``PyObject_HEAD_INIT`` macro. For :ref:`statically allocated objects - `, these fields always remain ``NULL``. For :ref:`dynamically - allocated objects `, these two fields are used to link the - object into a doubly linked list of *all* live objects on the heap. - - This could be used for various debugging purposes; currently the only uses - are the :func:`sys.getobjects` function and to print the objects that are - still alive at the end of a run when the environment variable - :envvar:`PYTHONDUMPREFS` is set. - - **Inheritance:** - - These fields are not inherited by subtypes. - - PyVarObject Slots ----------------- @@ -569,8 +547,8 @@ PyTypeObject Slots Each slot has a section describing inheritance. If :c:func:`PyType_Ready` may set a value when the field is set to ``NULL`` then there will also be -a "Default" section. (Note that many fields set on :c:type:`PyBaseObject_Type` -and :c:type:`PyType_Type` effectively act as defaults.) +a "Default" section. (Note that many fields set on :c:data:`PyBaseObject_Type` +and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_name @@ -579,7 +557,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) name, followed by a dot, followed by the type name; for built-in types, it should be just the type name. If the module is a submodule of a package, the full package name is part of the full module name. For example, a type named - :class:`T` defined in module :mod:`M` in subpackage :mod:`Q` in package :mod:`P` + :class:`!T` defined in module :mod:`!M` in subpackage :mod:`!Q` in package :mod:`!P` should have the :c:member:`~PyTypeObject.tp_name` initializer ``"P.Q.M.T"``. For :ref:`dynamically allocated type objects `, @@ -619,20 +597,20 @@ and :c:type:`PyType_Type` effectively act as defaults.) instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`. For a type with variable-length instances, the instances must have an - :attr:`ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N + :c:member:`~PyVarObject.ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of - N is typically stored in the instance's :attr:`ob_size` field. There are - exceptions: for example, ints use a negative :attr:`ob_size` to indicate a + N is typically stored in the instance's :c:member:`~PyVarObject.ob_size` field. There are + exceptions: for example, ints use a negative :c:member:`~PyVarObject.ob_size` to indicate a negative number, and N is ``abs(ob_size)`` there. Also, the presence of an - :attr:`ob_size` field in the instance layout doesn't mean that the instance + :c:member:`~PyVarObject.ob_size` field in the instance layout doesn't mean that the instance structure is variable-length (for example, the structure for the list type has - fixed-length instances, yet those instances have a meaningful :attr:`ob_size` + fixed-length instances, yet those instances have a meaningful :c:member:`~PyVarObject.ob_size` field). The basic size includes the fields in the instance declared by the macro :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to - declare the instance struct) and this in turn includes the :attr:`_ob_prev` and - :attr:`_ob_next` fields if they are present. This means that the only correct + declare the instance struct) and this in turn includes the :c:member:`~PyObject._ob_prev` and + :c:member:`~PyObject._ob_next` fields if they are present. This means that the only correct way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the ``sizeof`` operator on the struct used to declare the instance layout. The basic size does not include the GC header size. @@ -669,15 +647,15 @@ and :c:type:`PyType_Type` effectively act as defaults.) memory buffers owned by the instance (using the freeing function corresponding to the allocation function used to allocate the buffer), and call the type's :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable - (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is + (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the instance; this is normally :c:func:`PyObject_Del` if the instance was allocated - using :c:func:`PyObject_New` or :c:func:`PyObject_VarNew`, or + using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or :c:func:`PyObject_GC_Del` if the instance was allocated using - :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`. + :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. - If the type supports garbage collection (has the :const:`Py_TPFLAGS_HAVE_GC` + If the type supports garbage collection (has the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack` before clearing any member fields. @@ -689,8 +667,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) Py_TYPE(self)->tp_free((PyObject *)self); } - Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should decrement the reference count for its type object after + Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should release the owned reference to its type object + (via :c:func:`Py_DECREF`) after calling the type deallocator. In order to avoid dangling pointers, the recommended way to achieve this is: @@ -716,12 +695,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) a more efficient alternative of the simpler :c:member:`~PyTypeObject.tp_call`. - This field is only used if the flag :const:`Py_TPFLAGS_HAVE_VECTORCALL` + This field is only used if the flag :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` is set. If so, this must be a positive integer containing the offset in the instance of a :c:type:`vectorcallfunc` pointer. The *vectorcallfunc* pointer may be ``NULL``, in which case the instance behaves - as if :const:`Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance + as if :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance falls back to :c:member:`~PyTypeObject.tp_call`. Any class that sets ``Py_TPFLAGS_HAVE_VECTORCALL`` must also set @@ -740,15 +719,15 @@ and :c:type:`PyType_Type` effectively act as defaults.) Before version 3.12, it was not recommended for :ref:`mutable heap types ` to implement the vectorcall protocol. - When a user sets :attr:`~type.__call__` in Python code, only *tp_call* is + When a user sets :attr:`~object.__call__` in Python code, only *tp_call* is updated, likely making it inconsistent with the vectorcall function. Since 3.12, setting ``__call__`` will disable vectorcall optimization - by clearing the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag. + by clearing the :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag. **Inheritance:** This field is always inherited. - However, the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag is not + However, the :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag is not always inherited. If it's not set, then the subclass won't use :ref:`vectorcall `, except when :c:func:`PyVectorcall_Call` is explicitly called. @@ -764,7 +743,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :attr:`tp_getattr`, :attr:`tp_getattro` + Group: :c:member:`~PyTypeObject.tp_getattr`, :c:member:`~PyTypeObject.tp_getattro` This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when @@ -781,7 +760,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :attr:`tp_setattr`, :attr:`tp_setattro` + Group: :c:member:`~PyTypeObject.tp_setattr`, :c:member:`~PyTypeObject.tp_setattro` This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when @@ -883,7 +862,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) normal return value; when an error occurs during the computation of the hash value, the function should set an exception and return ``-1``. - When this field is not set (*and* :attr:`tp_richcompare` is not set), + When this field is not set (*and* :c:member:`~PyTypeObject.tp_richcompare` is not set), an attempt to take the hash of the object raises :exc:`TypeError`. This is the same as setting it to :c:func:`PyObject_HashNotImplemented`. @@ -897,7 +876,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :attr:`tp_hash`, :attr:`tp_richcompare` + Group: :c:member:`~PyTypeObject.tp_hash`, :c:member:`~PyTypeObject.tp_richcompare` This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_richcompare`: a subtype inherits both of @@ -956,7 +935,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :attr:`tp_getattr`, :attr:`tp_getattro` + Group: :c:member:`~PyTypeObject.tp_getattr`, :c:member:`~PyTypeObject.tp_getattro` This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when @@ -964,7 +943,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - :c:type:`PyBaseObject_Type` uses :c:func:`PyObject_GenericGetAttr`. + :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericGetAttr`. .. c:member:: setattrofunc PyTypeObject.tp_setattro @@ -982,7 +961,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :attr:`tp_setattr`, :attr:`tp_setattro` + Group: :c:member:`~PyTypeObject.tp_setattr`, :c:member:`~PyTypeObject.tp_setattro` This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when @@ -990,7 +969,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - :c:type:`PyBaseObject_Type` uses :c:func:`PyObject_GenericSetAttr`. + :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericSetAttr`. .. c:member:: PyBufferProcs* PyTypeObject.tp_as_buffer @@ -1022,30 +1001,32 @@ and :c:type:`PyType_Type` effectively act as defaults.) this flag bit. The flag bits that pertain to extension structures are strictly inherited if the extension structure is inherited, i.e. the base type's value of the flag bit is copied into the subtype together with a pointer to the extension - structure. The :const:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with + structure. The :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the - :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL`` values. .. XXX are most flag bits *really* inherited individually? **Default:** - :c:type:`PyBaseObject_Type` uses + :c:data:`PyBaseObject_Type` uses ``Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE``. **Bit Masks:** + .. c:namespace:: NULL + The following bit masks are currently defined; these can be ORed together using the ``|`` operator to form the value of the :c:member:`~PyTypeObject.tp_flags` field. The macro :c:func:`PyType_HasFeature` takes a type and a flags value, *tp* and *f*, and checks whether ``tp->tp_flags & f`` is non-zero. - .. data:: Py_TPFLAGS_HEAPTYPE + .. c:macro:: Py_TPFLAGS_HEAPTYPE This bit is set when the type object itself is allocated on the heap, for example, types created dynamically using :c:func:`PyType_FromSpec`. In this - case, the :attr:`ob_type` field of its instances is considered a reference to + case, the :c:member:`~PyObject.ob_type` field of its instances is considered a reference to the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or @@ -1056,7 +1037,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) ??? - .. data:: Py_TPFLAGS_BASETYPE + .. c:macro:: Py_TPFLAGS_BASETYPE This bit is set when the type can be used as the base type of another type. If this bit is clear, the type cannot be subtyped (similar to a "final" class in @@ -1067,7 +1048,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) ??? - .. data:: Py_TPFLAGS_READY + .. c:macro:: Py_TPFLAGS_READY This bit is set when the type object has been fully initialized by :c:func:`PyType_Ready`. @@ -1077,7 +1058,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) ??? - .. data:: Py_TPFLAGS_READYING + .. c:macro:: Py_TPFLAGS_READYING This bit is set while :c:func:`PyType_Ready` is in the process of initializing the type object. @@ -1087,10 +1068,10 @@ and :c:type:`PyType_Type` effectively act as defaults.) ??? - .. data:: Py_TPFLAGS_HAVE_GC + .. c:macro:: Py_TPFLAGS_HAVE_GC This bit is set when the object supports garbage collection. If this bit - is set, instances must be created using :c:func:`PyObject_GC_New` and + is set, instances must be created using :c:macro:`PyObject_GC_New` and destroyed using :c:func:`PyObject_GC_Del`. More information in section :ref:`supporting-cycle-detection`. This bit also implies that the GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in @@ -1098,28 +1079,28 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :const:`Py_TPFLAGS_HAVE_GC`, :attr:`tp_traverse`, :attr:`tp_clear` + Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear` - The :const:`Py_TPFLAGS_HAVE_GC` flag bit is inherited - together with the :attr:`tp_traverse` and :attr:`tp_clear` - fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is - clear in the subtype and the :attr:`tp_traverse` and - :attr:`tp_clear` fields in the subtype exist and have ``NULL`` + The :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is inherited + together with the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` + fields, i.e. if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is + clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and + :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL`` values. - .. data:: Py_TPFLAGS_DEFAULT + .. c:macro:: Py_TPFLAGS_DEFAULT This is a bitmask of all the bits that pertain to the existence of certain fields in the type object and its extension structures. Currently, it includes - the following bits: :const:`Py_TPFLAGS_HAVE_STACKLESS_EXTENSION`. + the following bits: :c:macro:`Py_TPFLAGS_HAVE_STACKLESS_EXTENSION`. **Inheritance:** ??? - .. data:: Py_TPFLAGS_METHOD_DESCRIPTOR + .. c:macro:: Py_TPFLAGS_METHOD_DESCRIPTOR This bit indicates that objects behave like unbound methods. @@ -1140,17 +1121,20 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** This flag is never inherited by types without the - :const:`Py_TPFLAGS_IMMUTABLETYPE` flag set. For extension types, it is + :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag set. For extension types, it is inherited whenever :c:member:`~PyTypeObject.tp_descr_get` is inherited. - .. data:: Py_TPFLAGS_MANAGED_DICT + .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a ``__dict__`` - attribute, and that the space for the dictionary is managed by the VM. + This bit indicates that instances of the class have a ``__dict__`` + attribute, and that the space for the dictionary is managed by the VM. - If this flag is set, :const:`Py_TPFLAGS_HAVE_GC` should also be set. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. - .. versionadded:: 3.12 + The type traverse function must call :c:func:`PyObject_VisitManagedDict` + and its clear function must call :c:func:`PyObject_ClearManagedDict`. + + .. versionadded:: 3.12 **Inheritance:** @@ -1158,12 +1142,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_dictoffset` field is set in a superclass. - .. data:: Py_TPFLAGS_MANAGED_WEAKREF + .. c:macro:: Py_TPFLAGS_MANAGED_WEAKREF - This bit indicates that instances of the class should be weakly - referenceable. + This bit indicates that instances of the class should be weakly + referenceable. - .. versionadded:: 3.12 + .. versionadded:: 3.12 **Inheritance:** @@ -1171,14 +1155,14 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass. - .. data:: Py_TPFLAGS_ITEMS_AT_END + .. c:macro:: Py_TPFLAGS_ITEMS_AT_END Only usable with variable-size types, i.e. ones with non-zero - :c:member:`~PyObject.tp_itemsize`. + :c:member:`~PyTypeObject.tp_itemsize`. Indicates that the variable-sized portion of an instance of this type is at the end of the instance's memory area, at an offset of - :c:expr:`Py_TYPE(obj)->tp_basicsize` (which may be different in each + ``Py_TYPE(obj)->tp_basicsize`` (which may be different in each subclass). When setting this flag, be sure that all superclasses either @@ -1194,14 +1178,14 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. XXX Document more flags here? - .. data:: Py_TPFLAGS_LONG_SUBCLASS - .. data:: Py_TPFLAGS_LIST_SUBCLASS - .. data:: Py_TPFLAGS_TUPLE_SUBCLASS - .. data:: Py_TPFLAGS_BYTES_SUBCLASS - .. data:: Py_TPFLAGS_UNICODE_SUBCLASS - .. data:: Py_TPFLAGS_DICT_SUBCLASS - .. data:: Py_TPFLAGS_BASE_EXC_SUBCLASS - .. data:: Py_TPFLAGS_TYPE_SUBCLASS + .. c:macro:: Py_TPFLAGS_LONG_SUBCLASS + .. c:macro:: Py_TPFLAGS_LIST_SUBCLASS + .. c:macro:: Py_TPFLAGS_TUPLE_SUBCLASS + .. c:macro:: Py_TPFLAGS_BYTES_SUBCLASS + .. c:macro:: Py_TPFLAGS_UNICODE_SUBCLASS + .. c:macro:: Py_TPFLAGS_DICT_SUBCLASS + .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS + .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS These flags are used by functions such as :c:func:`PyLong_Check` to quickly determine if a type is a subclass @@ -1212,7 +1196,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) will behave differently depending on what kind of check is used. - .. data:: Py_TPFLAGS_HAVE_FINALIZE + .. c:macro:: Py_TPFLAGS_HAVE_FINALIZE This bit is set when the :c:member:`~PyTypeObject.tp_finalize` slot is present in the type structure. @@ -1225,7 +1209,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) type structure. - .. data:: Py_TPFLAGS_HAVE_VECTORCALL + .. c:macro:: Py_TPFLAGS_HAVE_VECTORCALL This bit is set when the class implements the :ref:`vectorcall protocol `. @@ -1245,7 +1229,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This flag can now be inherited by mutable classes. - .. data:: Py_TPFLAGS_IMMUTABLETYPE + .. c:macro:: Py_TPFLAGS_IMMUTABLETYPE This bit is set for type objects that are immutable: type attributes cannot be set nor deleted. @@ -1258,7 +1242,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.10 - .. data:: Py_TPFLAGS_DISALLOW_INSTANTIATION + .. c:macro:: Py_TPFLAGS_DISALLOW_INSTANTIATION Disallow creating instances of the type: set :c:member:`~PyTypeObject.tp_new` to NULL and don't create the ``__new__`` @@ -1289,7 +1273,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.10 - .. data:: Py_TPFLAGS_MAPPING + .. c:macro:: Py_TPFLAGS_MAPPING This bit indicates that instances of the class may match mapping patterns when used as the subject of a :keyword:`match` block. It is automatically @@ -1298,20 +1282,20 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. note:: - :const:`Py_TPFLAGS_MAPPING` and :const:`Py_TPFLAGS_SEQUENCE` are + :c:macro:`Py_TPFLAGS_MAPPING` and :c:macro:`Py_TPFLAGS_SEQUENCE` are mutually exclusive; it is an error to enable both flags simultaneously. **Inheritance:** This flag is inherited by types that do not already set - :const:`Py_TPFLAGS_SEQUENCE`. + :c:macro:`Py_TPFLAGS_SEQUENCE`. .. seealso:: :pep:`634` -- Structural Pattern Matching: Specification .. versionadded:: 3.10 - .. data:: Py_TPFLAGS_SEQUENCE + .. c:macro:: Py_TPFLAGS_SEQUENCE This bit indicates that instances of the class may match sequence patterns when used as the subject of a :keyword:`match` block. It is automatically @@ -1320,20 +1304,20 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. note:: - :const:`Py_TPFLAGS_MAPPING` and :const:`Py_TPFLAGS_SEQUENCE` are + :c:macro:`Py_TPFLAGS_MAPPING` and :c:macro:`Py_TPFLAGS_SEQUENCE` are mutually exclusive; it is an error to enable both flags simultaneously. **Inheritance:** This flag is inherited by types that do not already set - :const:`Py_TPFLAGS_MAPPING`. + :c:macro:`Py_TPFLAGS_MAPPING`. .. seealso:: :pep:`634` -- Structural Pattern Matching: Specification .. versionadded:: 3.10 - .. data:: Py_TPFLAGS_VALID_VERSION_TAG + .. c:macro:: Py_TPFLAGS_VALID_VERSION_TAG Internal. Do not set or unset this flag. To indicate that a class has changed call :c:func:`PyType_Modified` @@ -1357,7 +1341,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: traverseproc PyTypeObject.tp_traverse An optional pointer to a traversal function for the garbage collector. This is - only used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: int tp_traverse(PyObject *self, visitproc visit, void *arg); @@ -1367,8 +1351,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python - objects that the instance owns. For example, this is function :c:func:`local_traverse` from the - :mod:`_thread` extension module:: + objects that the instance owns. For example, this is function :c:func:`!local_traverse` from the + :mod:`!_thread` extension module:: static int local_traverse(localobject *self, visitproc visit, void *arg) @@ -1387,6 +1371,23 @@ and :c:type:`PyType_Type` effectively act as defaults.) debugging aid you may want to visit it anyway just so the :mod:`gc` module's :func:`~gc.get_referents` function will include it. + Heap types (:c:macro:`Py_TPFLAGS_HEAPTYPE`) must visit their type with:: + + Py_VISIT(Py_TYPE(self)); + + It is only needed since Python 3.9. To support Python 3.8 and older, this + line must be conditionnal:: + + #if PY_VERSION_HEX >= 0x03090000 + Py_VISIT(Py_TYPE(self)); + #endif + + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_VisitManagedDict` like this:: + + PyObject_VisitManagedDict((PyObject*)self, visit, arg); + .. warning:: When implementing :c:member:`~PyTypeObject.tp_traverse`, only the members that the instance *owns* (by having :term:`strong references @@ -1400,7 +1401,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) are allowed to be removed even if the instance is still alive). Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`local_traverse` to have these specific names; don't name them just + :c:func:`!local_traverse` to have these specific names; don't name them just anything. Instances of :ref:`heap-allocated types ` hold a reference to @@ -1419,10 +1420,10 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :const:`Py_TPFLAGS_HAVE_GC`, :attr:`tp_traverse`, :attr:`tp_clear` + Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear` This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_clear` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and + :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype. @@ -1430,7 +1431,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear An optional pointer to a clear function for the garbage collector. This is only - used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: int tp_clear(PyObject *); @@ -1459,9 +1460,10 @@ and :c:type:`PyType_Type` effectively act as defaults.) } The :c:func:`Py_CLEAR` macro should be used, because clearing references is - delicate: the reference to the contained object must not be decremented until + delicate: the reference to the contained object must not be released + (via :c:func:`Py_DECREF`) until after the pointer to the contained object is set to ``NULL``. This is because - decrementing the reference count may cause the contained object to become trash, + releasing the reference may cause the contained object to become trash, triggering a chain of reclamation activity that may include invoking arbitrary Python code (due to finalizers, or weakref callbacks, associated with the contained object). If it's possible for such code to reference *self* again, @@ -1469,6 +1471,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_ClearManagedDict` like this:: + + PyObject_ClearManagedDict((PyObject*)self); + Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called before an instance is deallocated. For example, when reference counting is enough to determine that an object is no longer used, the cyclic garbage @@ -1486,10 +1494,10 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :const:`Py_TPFLAGS_HAVE_GC`, :attr:`tp_traverse`, :attr:`tp_clear` + Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear` This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_traverse` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and + :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype. @@ -1511,21 +1519,23 @@ and :c:type:`PyType_Type` effectively act as defaults.) The following constants are defined to be used as the third argument for :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`: - +----------------+------------+ - | Constant | Comparison | - +================+============+ - | :const:`Py_LT` | ``<`` | - +----------------+------------+ - | :const:`Py_LE` | ``<=`` | - +----------------+------------+ - | :const:`Py_EQ` | ``==`` | - +----------------+------------+ - | :const:`Py_NE` | ``!=`` | - +----------------+------------+ - | :const:`Py_GT` | ``>`` | - +----------------+------------+ - | :const:`Py_GE` | ``>=`` | - +----------------+------------+ + .. c:namespace:: NULL + + +--------------------+------------+ + | Constant | Comparison | + +====================+============+ + | .. c:macro:: Py_LT | ``<`` | + +--------------------+------------+ + | .. c:macro:: Py_LE | ``<=`` | + +--------------------+------------+ + | .. c:macro:: Py_EQ | ``==`` | + +--------------------+------------+ + | .. c:macro:: Py_NE | ``!=`` | + +--------------------+------------+ + | .. c:macro:: Py_GT | ``>`` | + +--------------------+------------+ + | .. c:macro:: Py_GE | ``>=`` | + +--------------------+------------+ The following macro is defined to ease writing rich comparison functions: @@ -1537,7 +1547,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) they may be C ints or floats). The third argument specifies the requested operation, as for :c:func:`PyObject_RichCompare`. - The return value's reference count is properly incremented. + The returned value is a new :term:`strong reference`. On error, sets an exception and returns ``NULL`` from the function. @@ -1545,7 +1555,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - Group: :attr:`tp_hash`, :attr:`tp_richcompare` + Group: :c:member:`~PyTypeObject.tp_hash`, :c:member:`~PyTypeObject.tp_richcompare` This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_hash`: a subtype inherits :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` when @@ -1554,16 +1564,16 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - :c:type:`PyBaseObject_Type` provides a :attr:`tp_richcompare` + :c:data:`PyBaseObject_Type` provides a :c:member:`~PyTypeObject.tp_richcompare` implementation, which may be inherited. However, if only - :attr:`tp_hash` is defined, not even the inherited function is used + :c:member:`~PyTypeObject.tp_hash` is defined, not even the inherited function is used and instances of the type will not be able to participate in any comparisons. .. c:member:: Py_ssize_t PyTypeObject.tp_weaklistoffset - While this field is still supported, :const:`Py_TPFLAGS_MANAGED_WEAKREF` + While this field is still supported, :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` should be used instead, if at all possible. If the instances of this type are weakly referenceable, this field is greater @@ -1576,7 +1586,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for weak references to the type object itself. - It is an error to set both the :const:`Py_TPFLAGS_MANAGED_WEAKREF` bit and + It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and :c:member:`~PyTypeObject.tp_weaklist`. **Inheritance:** @@ -1588,7 +1598,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - If the :const:`Py_TPFLAGS_MANAGED_WEAKREF` bit is set in the + If the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit is set in the :c:member:`~PyTypeObject.tp_dict` field, then :c:member:`~PyTypeObject.tp_weaklistoffset` will be set to a negative value, to indicate that it is unsafe to use this field. @@ -1691,7 +1701,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) to a pointer, are valid C99 address constants. However, the unary '&' operator applied to a non-static variable - like :c:func:`PyBaseObject_Type` is not required to produce an address + like :c:data:`PyBaseObject_Type` is not required to produce an address constant. Compilers may support this (gcc does), MSVC does not. Both compilers are strictly standard conforming in this particular behavior. @@ -1717,7 +1727,19 @@ and :c:type:`PyType_Type` effectively act as defaults.) called; it may also be initialized to a dictionary containing initial attributes for the type. Once :c:func:`PyType_Ready` has initialized the type, extra attributes for the type may be added to this dictionary only if they don't - correspond to overloaded operations (like :meth:`__add__`). + correspond to overloaded operations (like :meth:`~object.__add__`). Once + initialization for the type has finished, this field should be + treated as read-only. + + Some types may not store their dictionary in this slot. + Use :c:func:`PyType_GetDict` to retrieve the dictionary for an arbitrary + type. + + .. versionchanged:: 3.12 + + Internals detail: For static builtin types, this is always ``NULL``. + Instead, the dict for such types is stored on ``PyInterpreterState``. + Use :c:func:`PyType_GetDict` to get the dict for an arbitrary type. **Inheritance:** @@ -1770,7 +1792,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: Py_ssize_t PyTypeObject.tp_dictoffset - While this field is still supported, :const:`Py_TPFLAGS_MANAGED_DICT` should be + While this field is still supported, :c:macro:`Py_TPFLAGS_MANAGED_DICT` should be used instead, if at all possible. If the instances of this type have a dictionary containing instance variables, @@ -1789,7 +1811,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) dictionary, so it is may be more efficient to call :c:func:`PyObject_GetAttr` when accessing an attribute on the object. - It is an error to set both the :const:`Py_TPFLAGS_MANAGED_WEAKREF` bit and + It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and :c:member:`~PyTypeObject.tp_dictoffset`. **Inheritance:** @@ -1797,15 +1819,15 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes. A subtype should not override this offset; doing so could be unsafe, if C code tries to access the dictionary at the previous offset. - To properly support inheritance, use :const:`Py_TPFLAGS_MANAGED_DICT`. + To properly support inheritance, use :c:macro:`Py_TPFLAGS_MANAGED_DICT`. **Default:** This slot has no default. For :ref:`static types `, if the - field is ``NULL`` then no :attr:`__dict__` gets created for instances. + field is ``NULL`` then no :attr:`~object.__dict__` gets created for instances. - If the :const:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_dict` field, then + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, then :c:member:`~PyTypeObject.tp_dictoffset` will be set to ``-1``, to indicate that it is unsafe to use this field. @@ -1814,10 +1836,10 @@ and :c:type:`PyType_Type` effectively act as defaults.) An optional pointer to an instance initialization function. - This function corresponds to the :meth:`__init__` method of classes. Like - :meth:`__init__`, it is possible to create an instance without calling - :meth:`__init__`, and it is possible to reinitialize an instance by calling its - :meth:`__init__` method again. + This function corresponds to the :meth:`~object.__init__` method of classes. Like + :meth:`!__init__`, it is possible to create an instance without calling + :meth:`!__init__`, and it is possible to reinitialize an instance by calling its + :meth:`!__init__` method again. The function signature is:: @@ -1825,7 +1847,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The self argument is the instance to be initialized; the *args* and *kwds* arguments represent positional and keyword arguments of the call to - :meth:`__init__`. + :meth:`~object.__init__`. The :c:member:`~PyTypeObject.tp_init` function, if not ``NULL``, is called when an instance is created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function @@ -1864,7 +1886,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:func:`PyType_GenericAlloc`, to force a standard heap allocation strategy. - For static subtypes, :c:type:`PyBaseObject_Type` uses + For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyType_GenericAlloc`. That is the recommended value for all statically defined types. @@ -1891,7 +1913,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) in :c:member:`~PyTypeObject.tp_new`, while for mutable types, most initialization should be deferred to :c:member:`~PyTypeObject.tp_init`. - Set the :const:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag to disallow creating + Set the :c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag to disallow creating instances of the type in Python. **Inheritance:** @@ -1925,9 +1947,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) In dynamic subtypes, this field is set to a deallocator suitable to match :c:func:`PyType_GenericAlloc` and the value of the - :const:`Py_TPFLAGS_HAVE_GC` flag bit. + :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. - For static subtypes, :c:type:`PyBaseObject_Type` uses PyObject_Del. + For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Del`. .. c:member:: inquiry PyTypeObject.tp_is_gc @@ -1936,7 +1958,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The garbage collector needs to know whether a particular object is collectible or not. Normally, it is sufficient to look at the object's type's - :c:member:`~PyTypeObject.tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But + :c:member:`~PyTypeObject.tp_flags` field, and check the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. But some types have a mixture of statically and dynamically allocated instances, and the statically allocated instances are not collectible. Such types should define this function; it should return ``1`` for a collectible instance, and @@ -1955,7 +1977,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** This slot has no default. If this field is ``NULL``, - :const:`Py_TPFLAGS_HAVE_GC` is used as the functional equivalent. + :c:macro:`Py_TPFLAGS_HAVE_GC` is used as the functional equivalent. .. c:member:: PyObject* PyTypeObject.tp_bases @@ -2102,7 +2124,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionchanged:: 3.8 Before version 3.8 it was necessary to set the - :const:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be + :c:macro:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be used. This is no longer required. .. seealso:: "Safe object finalization" (:pep:`442`) @@ -2114,7 +2136,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) In other words, it is used to implement :ref:`vectorcall ` for ``type.__call__``. If ``tp_vectorcall`` is ``NULL``, the default call implementation - using :attr:`__new__` and :attr:`__init__` is used. + using :meth:`~object.__new__` and :meth:`~object.__init__` is used. **Inheritance:** @@ -2123,7 +2145,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) -.. c:member:: char PyTypeObject.tp_watched +.. c:member:: unsigned char PyTypeObject.tp_watched Internal. Do not use. @@ -2150,7 +2172,7 @@ This results in types that are limited relative to types defined in Python: include any subinterpreter-specific state. Also, since :c:type:`PyTypeObject` is only part of the :ref:`Limited API -` as an opaque struct, any extension modules using static types must be +` as an opaque struct, any extension modules using static types must be compiled for a specific Python minor version. @@ -2161,7 +2183,7 @@ Heap Types An alternative to :ref:`static types ` is *heap-allocated types*, or *heap types* for short, which correspond closely to classes created by -Python's ``class`` statement. Heap types have the :const:`Py_TPFLAGS_HEAPTYPE` +Python's ``class`` statement. Heap types have the :c:macro:`Py_TPFLAGS_HEAPTYPE` flag set. This is done by filling a :c:type:`PyType_Spec` structure and calling @@ -2241,8 +2263,8 @@ Number Object Structures .. note:: - The :c:data:`nb_reserved` field should always be ``NULL``. It - was previously called :c:data:`nb_long`, and was renamed in + The :c:member:`~PyNumberMethods.nb_reserved` field should always be ``NULL``. It + was previously called :c:member:`!nb_long`, and was renamed in Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add @@ -2313,8 +2335,8 @@ Mapping Object Structures .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript This function is used by :c:func:`PyObject_SetItem`, - :c:func:`PyObject_DelItem`, :c:func:`PyObject_SetSlice` and - :c:func:`PyObject_DelSlice`. It has the same signature as + :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and + :c:func:`PySequence_DelSlice`. It has the same signature as :c:func:`!PyObject_SetItem`, but *v* can also be set to ``NULL`` to delete an item. If this slot is ``NULL``, the object does not support item assignment and deletion. @@ -2360,9 +2382,9 @@ Sequence Object Structures This slot must be filled for the :c:func:`PySequence_Check` function to return ``1``, it can be ``NULL`` otherwise. - Negative indexes are handled as follows: if the :attr:`sq_length` slot is + Negative indexes are handled as follows: if the :c:member:`~PySequenceMethods.sq_length` slot is filled, it is called and the sequence length is used to compute a positive - index which is passed to :attr:`sq_item`. If :attr:`sq_length` is ``NULL``, + index which is passed to :c:member:`~PySequenceMethods.sq_item`. If :c:member:`!sq_length` is ``NULL``, the index is passed as is to the function. .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item @@ -2425,7 +2447,7 @@ Buffer Object Structures Except for point (3), an implementation of this function MUST take these steps: - (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, + (1) Check if the request can be met. If not, raise :exc:`BufferError`, set :c:expr:`view->obj` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. @@ -2536,7 +2558,7 @@ Async Object Structures PyObject *am_aiter(PyObject *self); Must return an :term:`asynchronous iterator` object. - See :meth:`__anext__` for details. + See :meth:`~object.__anext__` for details. This slot may be set to ``NULL`` if an object does not implement asynchronous iteration protocol. @@ -2547,7 +2569,8 @@ Async Object Structures PyObject *am_anext(PyObject *self); - Must return an :term:`awaitable` object. See :meth:`__anext__` for details. + Must return an :term:`awaitable` object. + See :meth:`~object.__anext__` for details. This slot may be set to ``NULL``. .. c:member:: sendfunc PyAsyncMethods.am_send @@ -2572,8 +2595,8 @@ Slot Type typedefs The purpose of this function is to separate memory allocation from memory initialization. It should return a pointer to a block of memory of adequate length for the instance, suitably aligned, and initialized to zeros, but with - :attr:`ob_refcnt` set to ``1`` and :attr:`ob_type` set to the type argument. If - the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :attr:`ob_size` field + :c:member:`~PyObject.ob_refcnt` set to ``1`` and :c:member:`~PyObject.ob_type` set to the type argument. If + the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :c:member:`~PyVarObject.ob_size` field should be initialized to *nitems* and the length of the allocated memory block should be ``tp_basicsize + nitems*tp_itemsize``, rounded up to a multiple of ``sizeof(void*)``; otherwise, *nitems* is not used and the length of the block @@ -2769,7 +2792,7 @@ A type that supports weakrefs, instance dicts, and hashing:: A str subclass that cannot be subclassed and cannot be called to create instances (e.g. uses a separate factory func) using -:c:data:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag:: +:c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag:: typedef struct { PyUnicodeObject raw; diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index ab3a2e274d9395..5541eaa521803b 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -44,7 +44,7 @@ Python: .. c:type:: Py_UNICODE - This is a typedef of :c:expr:`wchar_t`, which is a 16-bit type or 32-bit type + This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type depending on the platform. .. versionchanged:: 3.3 @@ -52,6 +52,8 @@ Python: whether you selected a "narrow" or "wide" Unicode version of Python at build time. + .. deprecated-removed:: 3.13 3.15 + .. c:type:: PyASCIIObject PyCompactUnicodeObject @@ -73,19 +75,19 @@ Python: The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: -.. c:function:: int PyUnicode_Check(PyObject *o) +.. c:function:: int PyUnicode_Check(PyObject *obj) - Return true if the object *o* is a Unicode object or an instance of a Unicode + Return true if the object *obj* is a Unicode object or an instance of a Unicode subtype. This function always succeeds. -.. c:function:: int PyUnicode_CheckExact(PyObject *o) +.. c:function:: int PyUnicode_CheckExact(PyObject *obj) - Return true if the object *o* is a Unicode object, but not an instance of a + Return true if the object *obj* is a Unicode object, but not an instance of a subtype. This function always succeeds. -.. c:function:: int PyUnicode_READY(PyObject *o) +.. c:function:: int PyUnicode_READY(PyObject *unicode) Returns ``0``. This API is kept only for backward compatibility. @@ -95,17 +97,17 @@ access to internal read-only data of Unicode objects: This API does nothing since Python 3.12. -.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) +.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *unicode) - Return the length of the Unicode string, in code points. *o* has to be a + Return the length of the Unicode string, in code points. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 -.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *o) - Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *o) - Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *o) +.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *unicode) + Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *unicode) + Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *unicode) Return a pointer to the canonical representation cast to UCS1, UCS2 or UCS4 integer types for direct character access. No checks are performed if the @@ -127,18 +129,18 @@ access to internal read-only data of Unicode objects: ``PyUnicode_WCHAR_KIND`` has been removed. -.. c:function:: int PyUnicode_KIND(PyObject *o) +.. c:function:: int PyUnicode_KIND(PyObject *unicode) Return one of the PyUnicode kind constants (see above) that indicate how many - bytes per character this Unicode object uses to store its data. *o* has to + bytes per character this Unicode object uses to store its data. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 -.. c:function:: void* PyUnicode_DATA(PyObject *o) +.. c:function:: void* PyUnicode_DATA(PyObject *unicode) - Return a void pointer to the raw Unicode buffer. *o* has to be a Unicode + Return a void pointer to the raw Unicode buffer. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 @@ -166,25 +168,25 @@ access to internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *o, Py_ssize_t index) +.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) - Read a character from a Unicode object *o*, which must be in the "canonical" + Read a character from a Unicode object *unicode*, which must be in the "canonical" representation. This is less efficient than :c:func:`PyUnicode_READ` if you do multiple consecutive reads. .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *o) +.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *unicode) Return the maximum code point that is suitable for creating another string - based on *o*, which must be in the "canonical" representation. This is + based on *unicode*, which must be in the "canonical" representation. This is always an approximation but more efficient than iterating over the string. .. versionadded:: 3.3 -.. c:function:: int PyUnicode_IsIdentifier(PyObject *o) +.. c:function:: int PyUnicode_IsIdentifier(PyObject *unicode) Return ``1`` if the string is a valid identifier according to the language definition, section :ref:`identifiers`. Return ``0`` otherwise. @@ -270,25 +272,16 @@ These APIs can be used for fast direct character conversions: Return the character *ch* converted to lower case. - .. deprecated:: 3.3 - This function uses simple case mappings. - .. c:function:: Py_UCS4 Py_UNICODE_TOUPPER(Py_UCS4 ch) Return the character *ch* converted to upper case. - .. deprecated:: 3.3 - This function uses simple case mappings. - .. c:function:: Py_UCS4 Py_UNICODE_TOTITLE(Py_UCS4 ch) Return the character *ch* converted to title case. - .. deprecated:: 3.3 - This function uses simple case mappings. - .. c:function:: int Py_UNICODE_TODECIMAL(Py_UCS4 ch) @@ -365,9 +358,9 @@ APIs: .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *str, Py_ssize_t size) - Create a Unicode object from the char buffer *u*. The bytes will be + Create a Unicode object from the char buffer *str*. The bytes will be interpreted as being UTF-8 encoded. The buffer is copied into the new object. The return value might be a shared object, i.e. modification of the data is @@ -376,16 +369,16 @@ APIs: This function raises :exc:`SystemError` when: * *size* < 0, - * *u* is ``NULL`` and *size* > 0 + * *str* is ``NULL`` and *size* > 0 .. versionchanged:: 3.12 - *u* == ``NULL`` with *size* > 0 is not allowed anymore. + *str* == ``NULL`` with *size* > 0 is not allowed anymore. -.. c:function:: PyObject *PyUnicode_FromString(const char *u) +.. c:function:: PyObject *PyUnicode_FromString(const char *str) Create a Unicode object from a UTF-8 encoded null-terminated char buffer - *u*. + *str*. .. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) @@ -394,98 +387,149 @@ APIs: arguments, calculate the size of the resulting Python Unicode string and return a string with the values formatted into it. The variable arguments must be C types and must correspond exactly to the format characters in the *format* - ASCII-encoded string. The following format characters are allowed: - - .. % This should be exactly the same as the table in PyErr_Format. - - .. tabularcolumns:: |l|l|L| - - +-------------------+---------------------+----------------------------------+ - | Format Characters | Type | Comment | - +===================+=====================+==================================+ - | :attr:`%%` | *n/a* | The literal % character. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%c` | int | A single character, | - | | | represented as a C int. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%d` | int | Equivalent to | - | | | ``printf("%d")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%u` | unsigned int | Equivalent to | - | | | ``printf("%u")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%ld` | long | Equivalent to | - | | | ``printf("%ld")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%li` | long | Equivalent to | - | | | ``printf("%li")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lu` | unsigned long | Equivalent to | - | | | ``printf("%lu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lld` | long long | Equivalent to | - | | | ``printf("%lld")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lli` | long long | Equivalent to | - | | | ``printf("%lli")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%llu` | unsigned long long | Equivalent to | - | | | ``printf("%llu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zd` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zd")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zi` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zi")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zu` | size_t | Equivalent to | - | | | ``printf("%zu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%i` | int | Equivalent to | - | | | ``printf("%i")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%x` | int | Equivalent to | - | | | ``printf("%x")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%s` | const char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%p` | const void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%A` | PyObject\* | The result of calling | - | | | :func:`ascii`. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%U` | PyObject\* | A Unicode object. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%V` | PyObject\*, | A Unicode object (which may be | - | | const char\* | ``NULL``) and a null-terminated | - | | | C character array as a second | - | | | parameter (which will be used, | - | | | if the first parameter is | - | | | ``NULL``). | - +-------------------+---------------------+----------------------------------+ - | :attr:`%S` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Str`. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%R` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Repr`. | - +-------------------+---------------------+----------------------------------+ + ASCII-encoded string. + + A conversion specifier contains two or more characters and has the following + components, which must occur in this order: + + #. The ``'%'`` character, which marks the start of the specifier. + + #. Conversion flags (optional), which affect the result of some conversion + types. + + #. Minimum field width (optional). + If specified as an ``'*'`` (asterisk), the actual width is given in the + next argument, which must be of type :c:expr:`int`, and the object to + convert comes after the minimum field width and optional precision. + + #. Precision (optional), given as a ``'.'`` (dot) followed by the precision. + If specified as ``'*'`` (an asterisk), the actual precision is given in + the next argument, which must be of type :c:expr:`int`, and the value to + convert comes after the precision. + + #. Length modifier (optional). + + #. Conversion type. + + The conversion flag characters are: + + .. tabularcolumns:: |l|L| + + +-------+-------------------------------------------------------------+ + | Flag | Meaning | + +=======+=============================================================+ + | ``0`` | The conversion will be zero padded for numeric values. | + +-------+-------------------------------------------------------------+ + | ``-`` | The converted value is left adjusted (overrides the ``0`` | + | | flag if both are given). | + +-------+-------------------------------------------------------------+ + + The length modifiers for following integer conversions (``d``, ``i``, + ``o``, ``u``, ``x``, or ``X``) specify the type of the argument + (:c:expr:`int` by default): + + .. tabularcolumns:: |l|L| + + +----------+-----------------------------------------------------+ + | Modifier | Types | + +==========+=====================================================+ + | ``l`` | :c:expr:`long` or :c:expr:`unsigned long` | + +----------+-----------------------------------------------------+ + | ``ll`` | :c:expr:`long long` or :c:expr:`unsigned long long` | + +----------+-----------------------------------------------------+ + | ``j`` | :c:type:`intmax_t` or :c:type:`uintmax_t` | + +----------+-----------------------------------------------------+ + | ``z`` | :c:type:`size_t` or :c:type:`ssize_t` | + +----------+-----------------------------------------------------+ + | ``t`` | :c:type:`ptrdiff_t` | + +----------+-----------------------------------------------------+ + + The length modifier ``l`` for following conversions ``s`` or ``V`` specify + that the type of the argument is :c:expr:`const wchar_t*`. + + The conversion specifiers are: + + .. list-table:: + :widths: auto + :header-rows: 1 + + * - Conversion Specifier + - Type + - Comment + + * - ``%`` + - *n/a* + - The literal ``%`` character. + + * - ``d``, ``i`` + - Specified by the length modifier + - The decimal representation of a signed C integer. + + * - ``u`` + - Specified by the length modifier + - The decimal representation of an unsigned C integer. + + * - ``o`` + - Specified by the length modifier + - The octal representation of an unsigned C integer. + + * - ``x`` + - Specified by the length modifier + - The hexadecimal representation of an unsigned C integer (lowercase). + + * - ``X`` + - Specified by the length modifier + - The hexadecimal representation of an unsigned C integer (uppercase). + + * - ``c`` + - :c:expr:`int` + - A single character. + + * - ``s`` + - :c:expr:`const char*` or :c:expr:`const wchar_t*` + - A null-terminated C character array. + + * - ``p`` + - :c:expr:`const void*` + - The hex representation of a C pointer. + Mostly equivalent to ``printf("%p")`` except that it is guaranteed to + start with the literal ``0x`` regardless of what the platform's + ``printf`` yields. + + * - ``A`` + - :c:expr:`PyObject*` + - The result of calling :func:`ascii`. + + * - ``U`` + - :c:expr:`PyObject*` + - A Unicode object. + + * - ``V`` + - :c:expr:`PyObject*`, :c:expr:`const char*` or :c:expr:`const wchar_t*` + - A Unicode object (which may be ``NULL``) and a null-terminated + C character array as a second parameter (which will be used, + if the first parameter is ``NULL``). + + * - ``S`` + - :c:expr:`PyObject*` + - The result of calling :c:func:`PyObject_Str`. + + * - ``R`` + - :c:expr:`PyObject*` + - The result of calling :c:func:`PyObject_Repr`. .. note:: The width formatter unit is number of characters rather than bytes. - The precision formatter unit is number of bytes for ``"%s"`` and + The precision formatter unit is number of bytes or :c:type:`wchar_t` + items (if the length modifier ``l`` is used) for ``"%s"`` and ``"%V"`` (if the ``PyObject*`` argument is ``NULL``), and a number of characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` (if the ``PyObject*`` argument is not ``NULL``). - .. [1] For integer specifiers (d, u, ld, li, lu, lld, lli, llu, zd, zi, - zu, i, x): the 0-conversion flag has effect even when a precision is given. + .. note:: + Unlike to C :c:func:`printf` the ``0`` flag has effect even when + a precision is given for integer conversions (``d``, ``i``, ``u``, ``o``, + ``x``, or ``X``). .. versionchanged:: 3.2 Support for ``"%lld"`` and ``"%llu"`` added. @@ -498,6 +542,13 @@ APIs: ``"%V"``, ``"%S"``, ``"%R"`` added. .. versionchanged:: 3.12 + Support for conversion specifiers ``o`` and ``X``. + Support for length modifiers ``j`` and ``t``. + Length modifiers are now applied to all integer conversions. + Length modifier ``l`` is now applied to conversion specifiers ``s`` and ``V``. + Support for variable width and precision ``*``. + Support for flag ``-``. + An unrecognized format character now sets a :exc:`SystemError`. In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. @@ -513,7 +564,7 @@ APIs: Copy an instance of a Unicode subtype to a new true Unicode object if necessary. If *obj* is already a true Unicode object (not a subtype), - return the reference with incremented refcount. + return a new :term:`strong reference` to the object. Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. @@ -550,7 +601,7 @@ APIs: Py_ssize_t how_many) Copy characters from one Unicode object into another. This function performs - character conversion when necessary and falls back to :c:func:`memcpy` if + character conversion when necessary and falls back to :c:func:`!memcpy` if possible. Returns ``-1`` and sets an exception on error, otherwise returns the number of copied characters. @@ -595,29 +646,29 @@ APIs: .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_Substring(PyObject *str, Py_ssize_t start, \ +.. c:function:: PyObject* PyUnicode_Substring(PyObject *unicode, Py_ssize_t start, \ Py_ssize_t end) - Return a substring of *str*, from character index *start* (included) to + Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. .. versionadded:: 3.3 -.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *u, Py_UCS4 *buffer, \ +.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *unicode, Py_UCS4 *buffer, \ Py_ssize_t buflen, int copy_null) - Copy the string *u* into a UCS4 buffer, including a null character, if + Copy the string *unicode* into a UCS4 buffer, including a null character, if *copy_null* is set. Returns ``NULL`` and sets an exception on error (in particular, a :exc:`SystemError` if *buflen* is smaller than the length of - *u*). *buffer* is returned on success. + *unicode*). *buffer* is returned on success. .. versionadded:: 3.3 -.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u) +.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *unicode) - Copy the string *u* into a new UCS4 buffer that is allocated using + Copy the string *unicode* into a new UCS4 buffer that is allocated using :c:func:`PyMem_Malloc`. If this fails, ``NULL`` is returned with a :exc:`MemoryError` set. The returned buffer always has an extra null code point appended. @@ -632,7 +683,7 @@ The current locale encoding can be used to decode text from the operating system. .. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, \ - Py_ssize_t len, \ + Py_ssize_t length, \ const char *errors) Decode a string from UTF-8 on Android and VxWorks, or from the current @@ -663,7 +714,7 @@ system. .. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, const char *errors) Similar to :c:func:`PyUnicode_DecodeLocaleAndSize`, but compute the string - length using :c:func:`strlen`. + length using :c:func:`!strlen`. .. versionadded:: 3.3 @@ -737,7 +788,7 @@ conversion function: Accepts a :term:`path-like object`. -.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *str, Py_ssize_t size) Decode a string from the :term:`filesystem encoding and error handler`. @@ -753,7 +804,7 @@ conversion function: handler>` is now used. -.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) +.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *str) Decode a null-terminated string from the :term:`filesystem encoding and error handler`. @@ -788,21 +839,21 @@ conversion function: wchar_t Support """"""""""""""" -:c:expr:`wchar_t` support for platforms which support it: +:c:type:`wchar_t` support for platforms which support it: -.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *wstr, Py_ssize_t size) - Create a Unicode object from the :c:expr:`wchar_t` buffer *w* of the given *size*. + Create a Unicode object from the :c:type:`wchar_t` buffer *wstr* of the given *size*. Passing ``-1`` as the *size* indicates that the function must itself compute the length, - using wcslen. + using :c:func:`!wcslen`. Return ``NULL`` on failure. -.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) +.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *wstr, Py_ssize_t size) - Copy the Unicode object contents into the :c:expr:`wchar_t` buffer *w*. At most - *size* :c:expr:`wchar_t` characters are copied (excluding a possibly trailing - null termination character). Return the number of :c:expr:`wchar_t` characters + Copy the Unicode object contents into the :c:type:`wchar_t` buffer *wstr*. At most + *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing + null termination character). Return the number of :c:type:`wchar_t` characters copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*` string may or may not be null-terminated. It is the responsibility of the caller to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is @@ -816,12 +867,12 @@ wchar_t Support Convert the Unicode object to a wide character string. The output string always ends with a null character. If *size* is not ``NULL``, write the number of wide characters (excluding the trailing null termination character) into - *\*size*. Note that the resulting :c:expr:`wchar_t` string might contain + *\*size*. Note that the resulting :c:type:`wchar_t` string might contain null characters, which would cause the string to be truncated when used with most C functions. If *size* is ``NULL`` and the :c:expr:`wchar_t*` string contains null characters a :exc:`ValueError` is raised. - Returns a buffer allocated by :c:func:`PyMem_New` (use + Returns a buffer allocated by :c:macro:`PyMem_New` (use :c:func:`PyMem_Free` to free it) on success. On error, returns ``NULL`` and *\*size* is undefined. Raises a :exc:`MemoryError` if memory allocation is failed. @@ -864,10 +915,10 @@ Generic Codecs These are the generic codec APIs: -.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_Decode(const char *str, Py_ssize_t size, \ const char *encoding, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s*. + Create a Unicode object by decoding *size* bytes of the encoded string *str*. *encoding* and *errors* have the same meaning as the parameters of the same name in the :func:`str` built-in function. The codec to be used is looked up using the Python codec registry. Return ``NULL`` if an exception was raised by @@ -890,13 +941,13 @@ UTF-8 Codecs These are the UTF-8 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-8 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF8`. If @@ -920,8 +971,8 @@ These are the UTF-8 codec APIs: returned buffer always has an extra null byte appended (not included in *size*), regardless of whether there are any other null code points. - In the case of an error, ``NULL`` is returned with an exception set and no - *size* is stored. + On error, set an exception, set *size* to ``-1`` (if it's not NULL) and + return ``NULL``. This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not @@ -934,7 +985,7 @@ These are the UTF-8 codec APIs: The return type is now ``const char *`` rather of ``char *``. .. versionchanged:: 3.10 - This function is a part of the :ref:`limited API `. + This function is a part of the :ref:`limited API `. .. c:function:: const char* PyUnicode_AsUTF8(PyObject *unicode) @@ -953,7 +1004,7 @@ UTF-32 Codecs These are the UTF-32 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder) Decode *size* bytes from a UTF-32 encoded buffer string and return the @@ -980,7 +1031,7 @@ These are the UTF-32 codec APIs: Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF32`. If @@ -1003,7 +1054,7 @@ UTF-16 Codecs These are the UTF-16 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder) Decode *size* bytes from a UTF-16 encoded buffer string and return the @@ -1031,7 +1082,7 @@ These are the UTF-16 codec APIs: Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF16`. If @@ -1054,13 +1105,13 @@ UTF-7 Codecs These are the UTF-7 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-7 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF7`. If @@ -1075,11 +1126,11 @@ Unicode-Escape Codecs These are the "Unicode Escape" codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, \ +.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *str, \ Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded - string *s*. Return ``NULL`` if an exception was raised by the codec. + string *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode) @@ -1095,11 +1146,11 @@ Raw-Unicode-Escape Codecs These are the "Raw Unicode Escape" codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, \ +.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *str, \ Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape - encoded string *s*. Return ``NULL`` if an exception was raised by the codec. + encoded string *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) @@ -1116,10 +1167,10 @@ These are the Latin-1 codec APIs: Latin-1 corresponds to the first 256 Unicode ordinals and only these are accepted by the codecs during encoding. -.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Latin-1 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode) @@ -1136,10 +1187,10 @@ These are the ASCII codec APIs. Only 7-bit ASCII data is accepted. All other codes generate errors. -.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the ASCII encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode) @@ -1154,16 +1205,16 @@ Character Map Codecs This codec is special in that it can be used to implement many different codecs (and this is in fact what was done to obtain most of the standard codecs -included in the :mod:`encodings` package). The codec uses mappings to encode and +included in the :mod:`!encodings` package). The codec uses mappings to encode and decode characters. The mapping objects provided must support the -:meth:`__getitem__` mapping interface; dictionaries and sequences work well. +:meth:`~object.__getitem__` mapping interface; dictionaries and sequences work well. These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *str, Py_ssize_t length, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* + Create a Unicode object by decoding *size* bytes of the encoded string *str* using the given *mapping* object. Return ``NULL`` if an exception was raised by the codec. @@ -1190,7 +1241,7 @@ These are the mapping codec APIs: The following codec API is special in that maps Unicode to Unicode. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors) +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, PyObject *table, const char *errors) Translate a string by applying a character mapping table to it and return the resulting Unicode object. Return ``NULL`` if an exception was raised by the @@ -1199,7 +1250,7 @@ The following codec API is special in that maps Unicode to Unicode. The mapping table must map Unicode ordinal integers to Unicode ordinal integers or ``None`` (causing deletion of the character). - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries + Mapping tables need only provide the :meth:`~object.__getitem__` interface; dictionaries and sequences work well. Unmapped character ordinals (ones which cause a :exc:`LookupError`) are left untouched and are copied as-is. @@ -1215,13 +1266,13 @@ use the Win32 MBCS converters to implement the conversions. Note that MBCS (or DBCS) is a class of encodings, not just one. The target encoding is defined by the user settings on the machine running the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *str, Py_ssize_t size, const char *errors) - Create a Unicode object by decoding *size* bytes of the MBCS encoded string *s*. + Create a Unicode object by decoding *size* bytes of the MBCS encoded string *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeMBCS`. If @@ -1241,7 +1292,7 @@ the user settings on the machine running the codec. Encode the Unicode object using the specified code page and return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. Use - :c:data:`CP_ACP` code page to get the MBCS encoder. + :c:macro:`!CP_ACP` code page to get the MBCS encoder. .. versionadded:: 3.3 @@ -1267,7 +1318,7 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Concat two strings giving a new Unicode string. -.. c:function:: PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) +.. c:function:: PyObject* PyUnicode_Split(PyObject *unicode, PyObject *sep, Py_ssize_t maxsplit) Split a string giving a list of Unicode strings. If *sep* is ``NULL``, splitting will be done at all whitespace substrings. Otherwise, splits occur at the given @@ -1275,10 +1326,10 @@ They all return ``NULL`` or ``-1`` if an exception occurs. set. Separators are not included in the resulting list. -.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *s, int keepend) +.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *unicode, int keepends) Split a Unicode string at line breaks, returning a list of Unicode strings. - CRLF is considered to be one line break. If *keepend* is ``0``, the line break + CRLF is considered to be one line break. If *keepends* is ``0``, the Line break characters are not included in the resulting strings. @@ -1288,28 +1339,28 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Unicode string. -.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return ``1`` if *substr* matches ``str[start:end]`` at the given tail end + Return ``1`` if *substr* matches ``unicode[start:end]`` at the given tail end (*direction* == ``-1`` means to do a prefix match, *direction* == ``1`` a suffix match), ``0`` otherwise. Return ``-1`` if an error occurred. -.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return the first position of *substr* in ``str[start:end]`` using the given + Return the first position of *substr* in ``unicode[start:end]`` using the given *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a backward search). The return value is the index of the first match; a value of ``-1`` indicates that no match was found, and ``-2`` indicates that an error occurred and an exception has been set. -.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, \ +.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *unicode, Py_UCS4 ch, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return the first position of the character *ch* in ``str[start:end]`` using + Return the first position of the character *ch* in ``unicode[start:end]`` using the given *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a backward search). The return value is the index of the first match; a value of ``-1`` indicates that no match was found, and ``-2`` @@ -1318,20 +1369,20 @@ They all return ``NULL`` or ``-1`` if an exception occurs. .. versionadded:: 3.3 .. versionchanged:: 3.7 - *start* and *end* are now adjusted to behave like ``str[start:end]``. + *start* and *end* are now adjusted to behave like ``unicode[start:end]``. -.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end) Return the number of non-overlapping occurrences of *substr* in - ``str[start:end]``. Return ``-1`` if an error occurred. + ``unicode[start:end]``. Return ``-1`` if an error occurred. -.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, \ +.. c:function:: PyObject* PyUnicode_Replace(PyObject *unicode, PyObject *substr, \ PyObject *replstr, Py_ssize_t maxcount) - Replace at most *maxcount* occurrences of *substr* in *str* with *replstr* and + Replace at most *maxcount* occurrences of *substr* in *unicode* with *replstr* and return the resulting Unicode object. *maxcount* == ``-1`` means replace all occurrences. @@ -1345,9 +1396,31 @@ They all return ``NULL`` or ``-1`` if an exception occurs. :c:func:`PyErr_Occurred` to check for errors. -.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) +.. c:function:: int PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *string, Py_ssize_t size) + + Compare a Unicode object with a char buffer which is interpreted as + being UTF-8 or ASCII encoded and return true (``1``) if they are equal, + or false (``0``) otherwise. + If the Unicode object contains surrogate characters or + the C string is not valid UTF-8, false (``0``) is returned. + + This function does not raise exceptions. + + .. versionadded:: 3.13 + + +.. c:function:: int PyUnicode_EqualToUTF8(PyObject *unicode, const char *string) + + Similar to :c:func:`PyUnicode_EqualToUTF8AndSize`, but compute *string* + length using :c:func:`!strlen`. + If the Unicode object contains null characters, false (``0``) is returned. + + .. versionadded:: 3.13 + + +.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *unicode, const char *string) - Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less + Compare a Unicode object, *unicode*, with *string* and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as ISO-8859-1 if it contains non-ASCII characters. @@ -1355,16 +1428,16 @@ They all return ``NULL`` or ``-1`` if an exception occurs. This function does not raise exceptions. -.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) +.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) Rich compare two Unicode strings and return one of the following: * ``NULL`` in case an exception was raised - * :const:`Py_True` or :const:`Py_False` for successful comparisons - * :const:`Py_NotImplemented` in case the type combination is unknown + * :c:data:`Py_True` or :c:data:`Py_False` for successful comparisons + * :c:data:`Py_NotImplemented` in case the type combination is unknown - Possible values for *op* are :const:`Py_GT`, :const:`Py_GE`, :const:`Py_EQ`, - :const:`Py_NE`, :const:`Py_LT`, and :const:`Py_LE`. + Possible values for *op* are :c:macro:`Py_GT`, :c:macro:`Py_GE`, :c:macro:`Py_EQ`, + :c:macro:`Py_NE`, :c:macro:`Py_LT`, and :c:macro:`Py_LE`. .. c:function:: PyObject* PyUnicode_Format(PyObject *format, PyObject *args) @@ -1373,29 +1446,29 @@ They all return ``NULL`` or ``-1`` if an exception occurs. ``format % args``. -.. c:function:: int PyUnicode_Contains(PyObject *container, PyObject *element) +.. c:function:: int PyUnicode_Contains(PyObject *unicode, PyObject *substr) - Check whether *element* is contained in *container* and return true or false + Check whether *substr* is contained in *unicode* and return true or false accordingly. - *element* has to coerce to a one element Unicode string. ``-1`` is returned + *substr* has to coerce to a one element Unicode string. ``-1`` is returned if there was an error. -.. c:function:: void PyUnicode_InternInPlace(PyObject **string) +.. c:function:: void PyUnicode_InternInPlace(PyObject **p_unicode) - Intern the argument *\*string* in place. The argument must be the address of a + Intern the argument :c:expr:`*p_unicode` in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an - existing interned string that is the same as *\*string*, it sets *\*string* to - it (decrementing the reference count of the old string object and incrementing - the reference count of the interned string object), otherwise it leaves - *\*string* alone and interns it (incrementing its reference count). - (Clarification: even though there is a lot of talk about reference counts, think - of this function as reference-count-neutral; you own the object after the call + existing interned string that is the same as :c:expr:`*p_unicode`, it sets :c:expr:`*p_unicode` to + it (releasing the reference to the old string object and creating a new + :term:`strong reference` to the interned string object), otherwise it leaves + :c:expr:`*p_unicode` alone and interns it (creating a new :term:`strong reference`). + (Clarification: even though there is a lot of talk about references, think + of this function as reference-neutral; you own the object after the call if and only if you owned it before the call.) -.. c:function:: PyObject* PyUnicode_InternFromString(const char *v) +.. c:function:: PyObject* PyUnicode_InternFromString(const char *str) A combination of :c:func:`PyUnicode_FromString` and :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst index a805b564763c40..48ae54acebe887 100644 --- a/Doc/c-api/utilities.rst +++ b/Doc/c-api/utilities.rst @@ -17,5 +17,7 @@ and parsing function arguments and constructing Python values from C values. marshal.rst arg.rst conversion.rst + hash.rst reflection.rst codec.rst + perfmaps.rst diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 513856d8a48d70..324518c035096b 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -12,12 +12,12 @@ file or a buffer, but they will not let you interact in a more detailed way with the interpreter. Several of these functions accept a start symbol from the grammar as a -parameter. The available start symbols are :const:`Py_eval_input`, -:const:`Py_file_input`, and :const:`Py_single_input`. These are described +parameter. The available start symbols are :c:data:`Py_eval_input`, +:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described following the functions which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One -particular issue which needs to be handled carefully is that the :c:expr:`FILE` +particular issue which needs to be handled carefully is that the :c:type:`FILE` structure for different C libraries can be different and incompatible. Under Windows (at least), it is possible for dynamically linked extensions to actually use different libraries, so care should be taken that :c:expr:`FILE*` parameters @@ -167,6 +167,10 @@ the same library that the Python runtime is using. event loops, as done in the :file:`Modules/_tkinter.c` in the Python source code. + .. versionchanged:: 3.12 + This function is only called from the + :ref:`main interpreter `. + .. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) @@ -187,6 +191,10 @@ the same library that the Python runtime is using. :c:func:`PyMem_RawRealloc`, instead of being allocated by :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. + .. versionchanged:: 3.12 + This function is only called from the + :ref:`main interpreter `. + .. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyRun_StringFlags` below, leaving @@ -248,8 +256,8 @@ the same library that the Python runtime is using. Parse and compile the Python source code in *str*, returning the resulting code object. The start token is given by *start*; this can be used to constrain the - code which can be compiled and should be :const:`Py_eval_input`, - :const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by + code which can be compiled and should be :c:data:`Py_eval_input`, + :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. @@ -345,7 +353,7 @@ the same library that the Python runtime is using. executed, it is passed as ``PyCompilerFlags *flags``. In this case, ``from __future__ import`` can modify *flags*. - Whenever ``PyCompilerFlags *flags`` is ``NULL``, :attr:`cf_flags` is treated as + Whenever ``PyCompilerFlags *flags`` is ``NULL``, :c:member:`~PyCompilerFlags.cf_flags` is treated as equal to ``0``, and any modification due to ``from __future__ import`` is discarded. @@ -359,7 +367,7 @@ the same library that the Python runtime is using. initialized to ``PY_MINOR_VERSION``. The field is ignored by default, it is used if and only if - ``PyCF_ONLY_AST`` flag is set in *cf_flags*. + ``PyCF_ONLY_AST`` flag is set in :c:member:`~PyCompilerFlags.cf_flags`. .. versionchanged:: 3.8 Added *cf_feature_version* field. diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index f27ec4411b4a26..038f54a9751fd1 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -11,20 +11,20 @@ simple reference object, and the second acts as a proxy for the original object as much as it can. -.. c:function:: int PyWeakref_Check(ob) +.. c:function:: int PyWeakref_Check(PyObject *ob) - Return true if *ob* is either a reference or proxy object. This function + Return non-zero if *ob* is either a reference or proxy object. This function always succeeds. -.. c:function:: int PyWeakref_CheckRef(ob) +.. c:function:: int PyWeakref_CheckRef(PyObject *ob) - Return true if *ob* is a reference object. This function always succeeds. + Return non-zero if *ob* is a reference object. This function always succeeds. -.. c:function:: int PyWeakref_CheckProxy(ob) +.. c:function:: int PyWeakref_CheckProxy(PyObject *ob) - Return true if *ob* is a proxy object. This function always succeeds. + Return non-zero if *ob* is a proxy object. This function always succeeds. .. c:function:: PyObject* PyWeakref_NewRef(PyObject *ob, PyObject *callback) @@ -51,10 +51,23 @@ as much as it can. ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. +.. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) + + Get a :term:`strong reference` to the referenced object from a weak + reference, *ref*, into *\*pobj*. + + * On success, set *\*pobj* to a new :term:`strong reference` to the + referenced object and return 1. + * If the reference is dead, set *\*pobj* to ``NULL`` and return 0. + * On error, raise an exception and return -1. + + .. versionadded:: 3.13 + + .. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) - Return the referenced object from a weak reference, *ref*. If the referent is - no longer live, returns :const:`Py_None`. + Return a :term:`borrowed reference` to the referenced object from a weak + reference, *ref*. If the referent is no longer live, returns ``Py_None``. .. note:: @@ -63,11 +76,17 @@ as much as it can. except when it cannot be destroyed before the last usage of the borrowed reference. + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyWeakref_GetRef` instead. + .. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) Similar to :c:func:`PyWeakref_GetObject`, but does no error checking. + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyWeakref_GetRef` instead. + .. c:function:: void PyObject_ClearWeakRefs(PyObject *object) diff --git a/Doc/conf.py b/Doc/conf.py index 485c0bdf84df2e..f2d36fdc70430c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -24,7 +24,13 @@ 'sphinx.ext.doctest', ] -# Skip if downstream redistributors haven't installed it +# Skip if downstream redistributors haven't installed them +try: + import notfound.extension +except ImportError: + pass +else: + extensions.append('notfound.extension') try: import sphinxext.opengraph except ImportError: @@ -66,7 +72,7 @@ highlight_language = 'python3' # Minimum version of sphinx required -needs_sphinx = '3.2' +needs_sphinx = '4.2' # Ignore any .rst files in the includes/ directory; # they're embedded in pages but not rendered individually. @@ -77,12 +83,179 @@ exclude_patterns.append(venvdir + '/*') nitpick_ignore = [ + # Standard C functions + ('c:func', 'calloc'), + ('c:func', 'dlopen'), + ('c:func', 'exec'), + ('c:func', 'fcntl'), + ('c:func', 'fork'), + ('c:func', 'free'), + ('c:func', 'gmtime'), + ('c:func', 'localtime'), + ('c:func', 'main'), + ('c:func', 'malloc'), + ('c:func', 'printf'), + ('c:func', 'realloc'), + ('c:func', 'snprintf'), + ('c:func', 'sprintf'), + ('c:func', 'stat'), + ('c:func', 'system'), + ('c:func', 'time'), + ('c:func', 'vsnprintf'), + # Standard C types + ('c:type', 'FILE'), + ('c:type', 'int64_t'), + ('c:type', 'intmax_t'), + ('c:type', 'off_t'), + ('c:type', 'ptrdiff_t'), + ('c:type', 'siginfo_t'), + ('c:type', 'size_t'), + ('c:type', 'ssize_t'), + ('c:type', 'time_t'), + ('c:type', 'uint64_t'), + ('c:type', 'uintmax_t'), + ('c:type', 'uintptr_t'), + ('c:type', 'va_list'), + ('c:type', 'wchar_t'), + ('c:type', '__int64'), + ('c:type', 'unsigned __int64'), + # Standard C structures + ('c:struct', 'in6_addr'), + ('c:struct', 'in_addr'), + ('c:struct', 'stat'), + ('c:struct', 'statvfs'), + # Standard C macros + ('c:macro', 'LLONG_MAX'), + ('c:macro', 'LLONG_MIN'), + ('c:macro', 'LONG_MAX'), + ('c:macro', 'LONG_MIN'), + # Standard C variables + ('c:data', 'errno'), + # Standard environment variables + ('envvar', 'BROWSER'), + ('envvar', 'COLUMNS'), + ('envvar', 'COMSPEC'), + ('envvar', 'DISPLAY'), + ('envvar', 'HOME'), + ('envvar', 'HOMEDRIVE'), + ('envvar', 'HOMEPATH'), + ('envvar', 'IDLESTARTUP'), + ('envvar', 'LANG'), + ('envvar', 'LANGUAGE'), + ('envvar', 'LC_ALL'), + ('envvar', 'LC_CTYPE'), + ('envvar', 'LC_COLLATE'), + ('envvar', 'LC_MESSAGES'), + ('envvar', 'LC_MONETARY'), + ('envvar', 'LC_NUMERIC'), + ('envvar', 'LC_TIME'), + ('envvar', 'LINES'), + ('envvar', 'LOGNAME'), + ('envvar', 'PAGER'), + ('envvar', 'PATH'), + ('envvar', 'PATHEXT'), + ('envvar', 'SOURCE_DATE_EPOCH'), + ('envvar', 'TEMP'), + ('envvar', 'TERM'), + ('envvar', 'TMP'), + ('envvar', 'TMPDIR'), + ('envvar', 'TZ'), + ('envvar', 'USER'), + ('envvar', 'USERNAME'), + ('envvar', 'USERPROFILE'), + # Deprecated function that was never documented: + ('py:func', 'getargspec'), + ('py:func', 'inspect.getargspec'), + # Undocumented modules that users shouldn't have to worry about + # (implementation details of `os.path`): + ('py:mod', 'ntpath'), + ('py:mod', 'posixpath'), +] + +# Temporary undocumented names. +# In future this list must be empty. +nitpick_ignore += [ + # C API: Standard Python exception classes + ('c:data', 'PyExc_ArithmeticError'), + ('c:data', 'PyExc_AssertionError'), + ('c:data', 'PyExc_AttributeError'), + ('c:data', 'PyExc_BaseException'), + ('c:data', 'PyExc_BlockingIOError'), + ('c:data', 'PyExc_BrokenPipeError'), + ('c:data', 'PyExc_BufferError'), + ('c:data', 'PyExc_ChildProcessError'), + ('c:data', 'PyExc_ConnectionAbortedError'), + ('c:data', 'PyExc_ConnectionError'), + ('c:data', 'PyExc_ConnectionRefusedError'), + ('c:data', 'PyExc_ConnectionResetError'), + ('c:data', 'PyExc_EOFError'), + ('c:data', 'PyExc_Exception'), + ('c:data', 'PyExc_FileExistsError'), + ('c:data', 'PyExc_FileNotFoundError'), + ('c:data', 'PyExc_FloatingPointError'), + ('c:data', 'PyExc_GeneratorExit'), + ('c:data', 'PyExc_ImportError'), + ('c:data', 'PyExc_IndentationError'), + ('c:data', 'PyExc_IndexError'), + ('c:data', 'PyExc_InterruptedError'), + ('c:data', 'PyExc_IsADirectoryError'), + ('c:data', 'PyExc_KeyboardInterrupt'), + ('c:data', 'PyExc_KeyError'), + ('c:data', 'PyExc_LookupError'), + ('c:data', 'PyExc_MemoryError'), + ('c:data', 'PyExc_ModuleNotFoundError'), + ('c:data', 'PyExc_NameError'), + ('c:data', 'PyExc_NotADirectoryError'), + ('c:data', 'PyExc_NotImplementedError'), + ('c:data', 'PyExc_OSError'), + ('c:data', 'PyExc_OverflowError'), + ('c:data', 'PyExc_PermissionError'), + ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_RecursionError'), + ('c:data', 'PyExc_ReferenceError'), + ('c:data', 'PyExc_RuntimeError'), + ('c:data', 'PyExc_StopAsyncIteration'), + ('c:data', 'PyExc_StopIteration'), + ('c:data', 'PyExc_SyntaxError'), + ('c:data', 'PyExc_SystemError'), + ('c:data', 'PyExc_SystemExit'), + ('c:data', 'PyExc_TabError'), + ('c:data', 'PyExc_TimeoutError'), + ('c:data', 'PyExc_TypeError'), + ('c:data', 'PyExc_UnboundLocalError'), + ('c:data', 'PyExc_UnicodeDecodeError'), + ('c:data', 'PyExc_UnicodeEncodeError'), + ('c:data', 'PyExc_UnicodeError'), + ('c:data', 'PyExc_UnicodeTranslateError'), + ('c:data', 'PyExc_ValueError'), + ('c:data', 'PyExc_ZeroDivisionError'), + # C API: Standard Python warning classes + ('c:data', 'PyExc_BytesWarning'), + ('c:data', 'PyExc_DeprecationWarning'), + ('c:data', 'PyExc_FutureWarning'), + ('c:data', 'PyExc_ImportWarning'), + ('c:data', 'PyExc_PendingDeprecationWarning'), + ('c:data', 'PyExc_ResourceWarning'), + ('c:data', 'PyExc_RuntimeWarning'), + ('c:data', 'PyExc_SyntaxWarning'), + ('c:data', 'PyExc_UnicodeWarning'), + ('c:data', 'PyExc_UserWarning'), + ('c:data', 'PyExc_Warning'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), ] +# gh-106948: Copy standard C types declared in the "c:type" domain to the +# "c:identifier" domain, since "c:function" markup looks for types in the +# "c:identifier" domain. Use list() to not iterate on items which are being +# added +for role, name in list(nitpick_ignore): + if role == 'c:type': + nitpick_ignore.append(('c:identifier', name)) +del role, name + # Disable Docutils smartquotes for several translations smartquotes_excludes = { 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], @@ -128,9 +301,8 @@ "pr_id": os.getenv("READTHEDOCS_VERSION") } -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' +# This 'Last updated on:' timestamp is inserted at the bottom of every page. +html_last_updated_fmt = time.strftime('%b %d, %Y (%H:%M UTC)', time.gmtime()) # Path to find HTML templates. templates_path = ['tools/templates'] @@ -194,8 +366,6 @@ latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), - ('distributing/index', 'distributing.tex', - 'Distributing Python Modules', _stdauthor, 'manual'), ('extending/index', 'extending.tex', 'Extending and Embedding Python', _stdauthor, 'manual'), ('installing/index', 'installing.tex', @@ -235,7 +405,6 @@ # match any of the following regexes (using re.match). coverage_ignore_modules = [ r'[T|t][k|K]', - r'Tix', ] coverage_ignore_functions = [ diff --git a/Doc/constraints.txt b/Doc/constraints.txt index 66c748eb092d83..147de1271eb2b7 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -10,8 +10,7 @@ colorama<0.5 imagesize<1.5 Jinja2<3.2 packaging<24 -# Pygments==2.15.0 breaks CI -Pygments<2.16,!=2.15.0 +Pygments>=2.16.1,<3 requests<3 snowballstemmer<3 sphinxcontrib-applehelp<1.1 @@ -23,7 +22,3 @@ sphinxcontrib-serializinghtml<1.2 # Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above) MarkupSafe<2.2 - -# Direct dependencies of sphinx-lint -polib<1.3 -regex<2024 diff --git a/Doc/contents.rst b/Doc/contents.rst index 464f93bdf85f95..24ceacb0076b5e 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -11,7 +11,6 @@ library/index.rst extending/index.rst c-api/index.rst - distributing/index.rst installing/index.rst howto/index.rst faq/index.rst @@ -21,10 +20,3 @@ bugs.rst copyright.rst license.rst - -.. to include legacy packaging docs in build - -.. toctree:: - :hidden: - - install/index.rst diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ee64ffdc91662e..ef9ac1617a284b 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -764,8 +764,6 @@ PyErr_WarnFormat::...:: PyErr_WriteUnraisable:void::: PyErr_WriteUnraisable:PyObject*:obj:0: -PyEval_AcquireLock:void::: - PyEval_AcquireThread:void::: PyEval_AcquireThread:PyThreadState*:tstate:: @@ -783,10 +781,6 @@ PyEval_GetFuncDesc:PyObject*:func:0: PyEval_GetFuncName:const char*::: PyEval_GetFuncName:PyObject*:func:0: -PyEval_InitThreads:void::: - -PyEval_ReleaseLock:void::: - PyEval_ReleaseThread:void::: PyEval_ReleaseThread:PyThreadState*:tstate:: @@ -980,6 +974,9 @@ PyCoro_New:PyFrameObject*:frame:0: PyCoro_New:PyObject*:name:0: PyCoro_New:PyObject*:qualname:0: +PyImport_AddModuleRef:PyObject*::+1: +PyImport_AddModuleRef:const char*:name:: + PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules PyImport_AddModule:const char*:name:: @@ -1584,21 +1581,6 @@ PyOS_FSPath:PyObject*:path:0: PyObject_ASCII:PyObject*::+1: PyObject_ASCII:PyObject*:o:0: -PyObject_AsCharBuffer:int::: -PyObject_AsCharBuffer:PyObject*:obj:0: -PyObject_AsCharBuffer:const char**:buffer:: -PyObject_AsCharBuffer:Py_ssize_t*:buffer_len:: - -PyObject_AsReadBuffer:int::: -PyObject_AsReadBuffer:PyObject*:obj:0: -PyObject_AsReadBuffer:const void**:buffer:: -PyObject_AsReadBuffer:Py_ssize_t*:buffer_len:: - -PyObject_AsWriteBuffer:int::: -PyObject_AsWriteBuffer:PyObject*:obj:0: -PyObject_AsWriteBuffer:void**:buffer:: -PyObject_AsWriteBuffer:Py_ssize_t*:buffer_len:: - PyObject_Bytes:PyObject*::+1: PyObject_Bytes:PyObject*:o:0: @@ -1815,6 +1797,9 @@ PyObject_Size:PyObject*:o:0: PyObject_Str:PyObject*::+1: PyObject_Str:PyObject*:o:0: +Py_TYPE:PyObject*::0: +Py_TYPE:PyObject*:ob:0: + PyObject_Type:PyObject*::+1: PyObject_Type:PyObject*:o:0: @@ -2374,76 +2359,56 @@ PyUnicode_KIND:PyObject*:o:0: PyUnicode_MAX_CHAR_VALUE:::: PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0: -PyUnicode_AS_UNICODE:Py_UNICODE*::: -PyUnicode_AS_UNICODE:PyObject*:o:0: - -PyUnicode_AS_DATA:const char*::: -PyUnicode_AS_DATA:PyObject*:o:0: - Py_UNICODE_ISALNUM:int::: -Py_UNICODE_ISALNUM:Py_UNICODE:ch:: +Py_UNICODE_ISALNUM:Py_UCS4:ch:: Py_UNICODE_ISALPHA:int::: -Py_UNICODE_ISALPHA:Py_UNICODE:ch:: +Py_UNICODE_ISALPHA:Py_UCS4:ch:: Py_UNICODE_ISSPACE:int::: -Py_UNICODE_ISSPACE:Py_UNICODE:ch:: +Py_UNICODE_ISSPACE:Py_UCS4:ch:: Py_UNICODE_ISLOWER:int::: -Py_UNICODE_ISLOWER:Py_UNICODE:ch:: +Py_UNICODE_ISLOWER:Py_UCS4:ch:: Py_UNICODE_ISUPPER:int::: -Py_UNICODE_ISUPPER:Py_UNICODE:ch:: +Py_UNICODE_ISUPPER:Py_UCS4:ch:: Py_UNICODE_ISTITLE:int::: -Py_UNICODE_ISTITLE:Py_UNICODE:ch:: +Py_UNICODE_ISTITLE:Py_UCS4:ch:: Py_UNICODE_ISLINEBREAK:int::: -Py_UNICODE_ISLINEBREAK:Py_UNICODE:ch:: +Py_UNICODE_ISLINEBREAK:Py_UCS4:ch:: Py_UNICODE_ISDECIMAL:int::: -Py_UNICODE_ISDECIMAL:Py_UNICODE:ch:: +Py_UNICODE_ISDECIMAL:Py_UCS4:ch:: Py_UNICODE_ISDIGIT:int::: -Py_UNICODE_ISDIGIT:Py_UNICODE:ch:: +Py_UNICODE_ISDIGIT:Py_UCS4:ch:: Py_UNICODE_ISNUMERIC:int::: -Py_UNICODE_ISNUMERIC:Py_UNICODE:ch:: +Py_UNICODE_ISNUMERIC:Py_UCS4:ch:: Py_UNICODE_ISPRINTABLE:int::: -Py_UNICODE_ISPRINTABLE:Py_UNICODE:ch:: +Py_UNICODE_ISPRINTABLE:Py_UCS4:ch:: -Py_UNICODE_TOLOWER:Py_UNICODE::: -Py_UNICODE_TOLOWER:Py_UNICODE:ch:: +Py_UNICODE_TOLOWER:Py_UCS4::: +Py_UNICODE_TOLOWER:Py_UCS4:ch:: -Py_UNICODE_TOUPPER:Py_UNICODE::: -Py_UNICODE_TOUPPER:Py_UNICODE:ch:: +Py_UNICODE_TOUPPER:Py_UCS4::: +Py_UNICODE_TOUPPER:Py_UCS4:ch:: -Py_UNICODE_TOTITLE:Py_UNICODE::: -Py_UNICODE_TOTITLE:Py_UNICODE:ch:: +Py_UNICODE_TOTITLE:Py_UCS4::: +Py_UNICODE_TOTITLE:Py_UCS4:ch:: Py_UNICODE_TODECIMAL:int::: -Py_UNICODE_TODECIMAL:Py_UNICODE:ch:: +Py_UNICODE_TODECIMAL:Py_UCS4:ch:: Py_UNICODE_TODIGIT:int::: -Py_UNICODE_TODIGIT:Py_UNICODE:ch:: +Py_UNICODE_TODIGIT:Py_UCS4:ch:: Py_UNICODE_TONUMERIC:double::: -Py_UNICODE_TONUMERIC:Py_UNICODE:ch:: - -PyUnicode_FromUnicode:PyObject*::+1: -PyUnicode_FromUnicode:const Py_UNICODE*:u:: -PyUnicode_FromUnicode:Py_ssize_t:size:: - -PyUnicode_AsUnicode:Py_UNICODE*::: -PyUnicode_AsUnicode:PyObject*:unicode:0: - -PyUnicode_AsUnicodeAndSize:Py_UNICODE*::: -PyUnicode_AsUnicodeAndSize:PyObject*:unicode:0: -PyUnicode_AsUnicodeAndSize:Py_ssize_t*:size:: - -PyUnicode_GetSize:Py_ssize_t::: -PyUnicode_GetSize:PyObject*:unicode:0: +Py_UNICODE_TONUMERIC:Py_UCS4:ch:: PyUnicode_FromObject:PyObject*::+1: PyUnicode_FromObject:PyObject*:obj:0: @@ -2848,6 +2813,10 @@ PyWeakref_GET_OBJECT:PyObject*:ref:0: PyWeakref_GetObject:PyObject*::0: PyWeakref_GetObject:PyObject*:ref:0: +PyWeakref_GetRef:int::: +PyWeakref_GetRef:PyObject*:ref:0: +PyWeakref_GetRef:PyObject**:pobj:+1: + PyWeakref_NewProxy:PyObject*::+1: PyWeakref_NewProxy:PyObject*:ob:0: PyWeakref_NewProxy:PyObject*:callback:0: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index f112d268129fd1..811b1bd84d2417 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -43,7 +43,6 @@ function,PyBytes_Size,3.2,, var,PyBytes_Type,3.2,, type,PyCFunction,3.2,, type,PyCFunctionWithKeywords,3.2,, -function,PyCFunction_Call,3.2,, function,PyCFunction_GetFlags,3.2,, function,PyCFunction_GetFunction,3.2,, function,PyCFunction_GetSelf,3.2,, @@ -112,7 +111,9 @@ function,PyDict_Copy,3.2,, function,PyDict_DelItem,3.2,, function,PyDict_DelItemString,3.2,, function,PyDict_GetItem,3.2,, +function,PyDict_GetItemRef,3.13,, function,PyDict_GetItemString,3.2,, +function,PyDict_GetItemStringRef,3.13,, function,PyDict_GetItemWithError,3.2,, function,PyDict_Items,3.2,, function,PyDict_Keys,3.2,, @@ -178,11 +179,7 @@ function,PyErr_WarnEx,3.2,, function,PyErr_WarnExplicit,3.2,, function,PyErr_WarnFormat,3.2,, function,PyErr_WriteUnraisable,3.2,, -function,PyEval_AcquireLock,3.2,, function,PyEval_AcquireThread,3.2,, -function,PyEval_CallFunction,3.2,, -function,PyEval_CallMethod,3.2,, -function,PyEval_CallObjectWithKeywords,3.2,, function,PyEval_EvalCode,3.2,, function,PyEval_EvalCodeEx,3.2,, function,PyEval_EvalFrame,3.2,, @@ -193,12 +190,9 @@ function,PyEval_GetFuncDesc,3.2,, function,PyEval_GetFuncName,3.2,, function,PyEval_GetGlobals,3.2,, function,PyEval_GetLocals,3.2,, -function,PyEval_InitThreads,3.2,, -function,PyEval_ReleaseLock,3.2,, function,PyEval_ReleaseThread,3.2,, function,PyEval_RestoreThread,3.2,, function,PyEval_SaveThread,3.2,, -function,PyEval_ThreadsInitialized,3.2,, var,PyExc_ArithmeticError,3.2,, var,PyExc_AssertionError,3.2,, var,PyExc_AttributeError,3.2,, @@ -306,6 +300,7 @@ type,PyGetSetDef,3.2,,full-abi var,PyGetSetDescr_Type,3.2,, function,PyImport_AddModule,3.2,, function,PyImport_AddModuleObject,3.7,, +function,PyImport_AddModuleRef,3.13,, function,PyImport_AppendInittab,3.2,, function,PyImport_ExecCodeModule,3.2,, function,PyImport_ExecCodeModuleEx,3.2,, @@ -352,6 +347,7 @@ var,PyList_Type,3.2,, type,PyLongObject,3.2,,opaque var,PyLongRangeIter_Type,3.2,, function,PyLong_AsDouble,3.2,, +function,PyLong_AsInt,3.13,, function,PyLong_AsLong,3.2,, function,PyLong_AsLongAndOverflow,3.2,, function,PyLong_AsLongLong,3.2,, @@ -377,8 +373,12 @@ var,PyLong_Type,3.2,, var,PyMap_Type,3.2,, function,PyMapping_Check,3.2,, function,PyMapping_GetItemString,3.2,, +function,PyMapping_GetOptionalItem,3.13,, +function,PyMapping_GetOptionalItemString,3.13,, function,PyMapping_HasKey,3.2,, function,PyMapping_HasKeyString,3.2,, +function,PyMapping_HasKeyStringWithError,3.13,, +function,PyMapping_HasKeyWithError,3.13,, function,PyMapping_Items,3.2,, function,PyMapping_Keys,3.2,, function,PyMapping_Length,3.2,, @@ -388,6 +388,10 @@ function,PyMapping_Values,3.2,, function,PyMem_Calloc,3.7,, function,PyMem_Free,3.2,, function,PyMem_Malloc,3.2,, +function,PyMem_RawCalloc,3.13,, +function,PyMem_RawFree,3.13,, +function,PyMem_RawMalloc,3.13,, +function,PyMem_RawRealloc,3.13,, function,PyMem_Realloc,3.2,, type,PyMemberDef,3.2,,full-abi var,PyMemberDescr_Type,3.2,, @@ -404,6 +408,7 @@ type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi function,PyModuleDef_Init,3.5,, var,PyModuleDef_Type,3.5,, +function,PyModule_Add,3.13,, function,PyModule_AddFunctions,3.7,, function,PyModule_AddIntConstant,3.2,, function,PyModule_AddObject,3.2,, @@ -484,10 +489,7 @@ type,PyObject,3.2,,members member,PyObject.ob_refcnt,3.2,, member,PyObject.ob_type,3.2,, function,PyObject_ASCII,3.2,, -function,PyObject_AsCharBuffer,3.2,, function,PyObject_AsFileDescriptor,3.2,, -function,PyObject_AsReadBuffer,3.2,, -function,PyObject_AsWriteBuffer,3.2,, function,PyObject_Bytes,3.2,, function,PyObject_Call,3.2,, function,PyObject_CallFunction,3.2,, @@ -498,9 +500,10 @@ function,PyObject_CallNoArgs,3.10,, function,PyObject_CallObject,3.2,, function,PyObject_Calloc,3.7,, function,PyObject_CheckBuffer,3.11,, -function,PyObject_CheckReadBuffer,3.2,, function,PyObject_ClearWeakRefs,3.2,, function,PyObject_CopyData,3.11,, +function,PyObject_DelAttr,3.13,, +function,PyObject_DelAttrString,3.13,, function,PyObject_DelItem,3.2,, function,PyObject_DelItemString,3.2,, function,PyObject_Dir,3.2,, @@ -521,9 +524,13 @@ function,PyObject_GetAttrString,3.2,, function,PyObject_GetBuffer,3.11,, function,PyObject_GetItem,3.2,, function,PyObject_GetIter,3.2,, +function,PyObject_GetOptionalAttr,3.13,, +function,PyObject_GetOptionalAttrString,3.13,, function,PyObject_GetTypeData,3.12,, function,PyObject_HasAttr,3.2,, function,PyObject_HasAttrString,3.2,, +function,PyObject_HasAttrStringWithError,3.13,, +function,PyObject_HasAttrWithError,3.13,, function,PyObject_Hash,3.2,, function,PyObject_HashNotImplemented,3.2,, function,PyObject_Init,3.2,, @@ -599,19 +606,14 @@ function,PyStructSequence_NewType,3.2,, function,PyStructSequence_SetItem,3.2,, var,PyStructSequence_UnnamedField,3.11,, var,PySuper_Type,3.2,, -function,PySys_AddWarnOption,3.2,, -function,PySys_AddWarnOptionUnicode,3.2,, -function,PySys_AddXOption,3.7,, +function,PySys_Audit,3.13,, +function,PySys_AuditTuple,3.13,, function,PySys_FormatStderr,3.2,, function,PySys_FormatStdout,3.2,, function,PySys_GetObject,3.2,, function,PySys_GetXOptions,3.7,, -function,PySys_HasWarnOptions,3.2,, function,PySys_ResetWarnOptions,3.2,, -function,PySys_SetArgv,3.2,, -function,PySys_SetArgvEx,3.2,, function,PySys_SetObject,3.2,, -function,PySys_SetPath,3.2,, function,PySys_WriteStderr,3.2,, function,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque @@ -759,6 +761,8 @@ function,PyUnicode_DecodeUnicodeEscape,3.2,, function,PyUnicode_EncodeCodePage,3.7,on Windows, function,PyUnicode_EncodeFSDefault,3.2,, function,PyUnicode_EncodeLocale,3.7,, +function,PyUnicode_EqualToUTF8,3.13,, +function,PyUnicode_EqualToUTF8AndSize,3.13,, function,PyUnicode_FSConverter,3.2,, function,PyUnicode_FSDecoder,3.2,, function,PyUnicode_Find,3.2,, @@ -799,6 +803,7 @@ function,PyVectorcall_Call,3.12,, function,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque function,PyWeakref_GetObject,3.2,, +function,PyWeakref_GetRef,3.13,, function,PyWeakref_NewProxy,3.2,, function,PyWeakref_NewRef,3.2,, var,PyWrapperDescr_Type,3.2,, @@ -843,6 +848,7 @@ function,Py_Initialize,3.2,, function,Py_InitializeEx,3.2,, function,Py_Is,3.10,, function,Py_IsFalse,3.10,, +function,Py_IsFinalizing,3.13,, function,Py_IsInitialized,3.2,, function,Py_IsNone,3.10,, function,Py_IsTrue,3.10,, @@ -853,9 +859,6 @@ function,Py_NewInterpreter,3.2,, function,Py_NewRef,3.10,, function,Py_ReprEnter,3.2,, function,Py_ReprLeave,3.2,, -function,Py_SetPath,3.7,, -function,Py_SetProgramName,3.2,, -function,Py_SetPythonHome,3.2,, function,Py_SetRecursionLimit,3.2,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index d237f8f082d87b..2430564b45d6d2 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -1,174 +1,19 @@ +:orphan: + +.. This page is retained solely for existing links to /distributing/index.html. + Direct readers to the PPUG instead. + .. _distributing-index: ############################### Distributing Python Modules ############################### -:Email: distutils-sig@python.org - - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the distribution part of the process. For a guide to -installing other Python projects, refer to the -:ref:`installation guide `. - .. note:: - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* the `Python Package Index `__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - `__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on `GitHub `__. -* ``distutils`` is the original build and distribution system first added - to the Python standard library in 1998. While direct use of ``distutils`` - is being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). -* `setuptools`_ is a (largely) drop-in replacement for ``distutils`` first - published in 2004. Its most notable addition over the unmodified - ``distutils`` tools was the ability to declare dependencies on other - packages. It is currently recommended as a more regularly updated - alternative to ``distutils`` that offers consistent support for more - recent packaging standards across a wide range of Python versions. -* `wheel`_ (in this context) is a project that adds the ``bdist_wheel`` - command to ``distutils``/`setuptools`_. This produces a cross platform - binary packaging format (called "wheels" or "wheel files" and defined in - :pep:`427`) that allows Python libraries, even those including binary - extensions, to be installed on a system without needing to be built - locally. - -.. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _wheel: https://wheel.readthedocs.io/ - -Open source licensing and collaboration -======================================= - -In most parts of the world, software is automatically covered by copyright. -This means that other developers require explicit permission to copy, use, -modify and redistribute the software. - -Open source licensing is a way of explicitly granting such permission in a -relatively consistent way, allowing developers to share and collaborate -efficiently by making common solutions to various problems freely available. -This leaves many developers free to spend more time focusing on the problems -that are relatively unique to their specific situation. - -The distribution tools provided with Python are designed to make it -reasonably straightforward for developers to make their own contributions -back to that common pool of software if they choose to do so. - -The same distribution tools can also be used to distribute software within -an organisation, regardless of whether that software is published as open -source software or not. - - -Installing the tools -==================== - -The standard library does not include build tools that support modern -Python packaging standards, as the core development team has found that it -is important to have standard tools that work consistently, even on older -versions of Python. - -The currently recommended build and distribution tools can be installed -by invoking the ``pip`` module at the command line:: - - python -m pip install setuptools wheel twine - -.. note:: - - For POSIX users (including macOS and Linux users), these instructions - assume the use of a :term:`virtual environment`. - - For Windows users, these instructions assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -The Python Packaging User Guide includes more details on the `currently -recommended tools`_. - -.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations - -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) - -.. _publishing-python-packages: - -Reading the Python Packaging User Guide -======================================= - -The Python Packaging User Guide covers the various key steps and elements -involved in creating and publishing a project: - -* `Project structure`_ -* `Building and packaging the project`_ -* `Uploading the project to the Python Package Index`_ -* `The .pypirc file`_ - -.. _Project structure: https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects -.. _Building and packaging the project: https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Package Index: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives -.. _The .pypirc file: https://packaging.python.org/specifications/pypirc/ - - -How do I...? -============ - -These are quick answers or links for some common tasks. - -... choose a name for my project? ---------------------------------- - -This isn't an easy topic, but here are a few tips: - -* check the Python Package Index to see if the name is already in use -* check popular hosting sites like GitHub, Bitbucket, etc to see if there - is already a project with that name -* check what comes up in a web search for the name you're considering -* avoid particularly common words, especially ones with multiple meanings, - as they can make it difficult for users to find your software when - searching for it - - -... create and distribute binary extensions? --------------------------------------------- - -This is actually quite a complex topic, with a variety of alternatives -available depending on exactly what you're aiming to achieve. See the -Python Packaging User Guide for more information and recommendations. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - `__ - -.. other topics: + Information and guidance on distributing Python modules and packages + has been moved to the `Python Packaging User Guide`_, + and the tutorial on `packaging Python projects`_. - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + .. _Python Packaging User Guide: https://packaging.python.org/ + .. _packaging Python projects: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index 880bb33ee56718..ddde567f6f3efa 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -45,6 +45,7 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. .. highlight:: c +.. _install-index: .. _setuptools-index: Building C and C++ Extensions with setuptools diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index e64db373344038..20397dc5add5db 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -59,24 +59,43 @@ perform some operation on a file. :: int main(int argc, char *argv[]) { - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); + PyStatus status; + PyConfig config; + PyConfig_InitPythonConfig(&config); + + /* optional but recommended */ + status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]); + if (PyStatus_Exception(status)) { + goto exception; } - Py_SetProgramName(program); /* optional but recommended */ - Py_Initialize(); + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; + } + PyConfig_Clear(&config); + PyRun_SimpleString("from time import time,ctime\n" "print('Today is', ctime(time()))\n"); if (Py_FinalizeEx() < 0) { exit(120); } - PyMem_RawFree(program); return 0; + + exception: + PyConfig_Clear(&config); + Py_ExitStatusException(status); } -The :c:func:`Py_SetProgramName` function should be called before -:c:func:`Py_Initialize` to inform the interpreter about paths to Python run-time +.. note:: + + ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be + used in some APIs instead of ``int``. + It is not necessary since Python 3.13, but we keep it here for backward compatibility. + See :ref:`arg-parsing-string-and-buffers` for a description of this macro. + +Setting :c:member:`PyConfig.program_name` should be called before +:c:func:`Py_InitializeFromConfig` to inform the interpreter about paths to Python run-time libraries. Next, the Python interpreter is initialized with :c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts @@ -250,7 +269,7 @@ following two statements before the call to :c:func:`Py_Initialize`:: PyImport_AppendInittab("emb", &PyInit_emb); These two lines initialize the ``numargs`` variable, and make the -:func:`emb.numargs` function accessible to the embedded Python interpreter. +:func:`!emb.numargs` function accessible to the embedded Python interpreter. With these extensions, the Python script can do things like .. code-block:: python diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index d9bf4fd6c7ae0e..745fc10a22d161 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -69,8 +69,10 @@ the module and a copyright notice if you like). headers on some systems, you *must* include :file:`Python.h` before any standard headers are included. - It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including - ``Python.h``. See :ref:`parsetuple` for a description of this macro. + ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be + used in some APIs instead of ``int``. + It is not necessary since Python 3.13, but we keep it here for backward compatibility. + See :ref:`arg-parsing-string-and-buffers` for a description of this macro. All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or ``PY``, except those defined in standard header files. For convenience, and @@ -195,7 +197,7 @@ The choice of which exception to raise is entirely yours. There are predeclared C objects corresponding to all built-in Python exceptions, such as :c:data:`PyExc_ZeroDivisionError`, which you can use directly. Of course, you should choose exceptions wisely --- don't use :c:data:`PyExc_TypeError` to mean -that a file couldn't be opened (that should probably be :c:data:`PyExc_IOError`). +that a file couldn't be opened (that should probably be :c:data:`PyExc_OSError`). If something's wrong with the argument list, the :c:func:`PyArg_ParseTuple` function usually raises :c:data:`PyExc_TypeError`. If you have an argument whose value must be in a particular range or must satisfy other conditions, @@ -206,7 +208,7 @@ usually declare a static object variable at the beginning of your file:: static PyObject *SpamError; -and initialize it in your module's initialization function (:c:func:`PyInit_spam`) +and initialize it in your module's initialization function (:c:func:`!PyInit_spam`) with an exception object:: PyMODINIT_FUNC @@ -219,9 +221,7 @@ with an exception object:: return NULL; SpamError = PyErr_NewException("spam.error", NULL, NULL); - Py_XINCREF(SpamError); - if (PyModule_AddObject(m, "error", SpamError) < 0) { - Py_XDECREF(SpamError); + if (PyModule_AddObjectRef(m, "error", SpamError) < 0) { Py_CLEAR(SpamError); Py_DECREF(m); return NULL; @@ -230,22 +230,22 @@ with an exception object:: return m; } -Note that the Python name for the exception object is :exc:`spam.error`. The +Note that the Python name for the exception object is :exc:`!spam.error`. The :c:func:`PyErr_NewException` function may create a class with the base class being :exc:`Exception` (unless another class is passed in instead of ``NULL``), described in :ref:`bltin-exceptions`. -Note also that the :c:data:`SpamError` variable retains a reference to the newly +Note also that the :c:data:`!SpamError` variable retains a reference to the newly created exception class; this is intentional! Since the exception could be removed from the module by external code, an owned reference to the class is -needed to ensure that it will not be discarded, causing :c:data:`SpamError` to +needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. -We discuss the use of ``PyMODINIT_FUNC`` as a function return type later in this +We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. -The :exc:`spam.error` exception can be raised in your extension module using a +The :exc:`!spam.error` exception can be raised in your extension module using a call to :c:func:`PyErr_SetString` as shown below:: static PyObject * @@ -279,9 +279,9 @@ statement:: It returns ``NULL`` (the error indicator for functions returning object pointers) if an error is detected in the argument list, relying on the exception set by :c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been -copied to the local variable :c:data:`command`. This is a pointer assignment and +copied to the local variable :c:data:`!command`. This is a pointer assignment and you are not supposed to modify the string to which it points (so in Standard C, -the variable :c:data:`command` should properly be declared as ``const char +the variable :c:data:`!command` should properly be declared as ``const char *command``). The next statement is a call to the Unix function :c:func:`system`, passing it @@ -289,7 +289,7 @@ the string we just got from :c:func:`PyArg_ParseTuple`:: sts = system(command); -Our :func:`spam.system` function must return the value of :c:data:`sts` as a +Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a Python object. This is done using the function :c:func:`PyLong_FromLong`. :: return PyLong_FromLong(sts); @@ -315,7 +315,7 @@ contexts, as we have seen. The Module's Method Table and Initialization Function ===================================================== -I promised to show how :c:func:`spam_system` is called from Python programs. +I promised to show how :c:func:`!spam_system` is called from Python programs. First, we need to list its name and address in a "method table":: static PyMethodDef SpamMethods[] = { @@ -335,7 +335,7 @@ When using only ``METH_VARARGS``, the function should expect the Python-level parameters to be passed in as a tuple acceptable for parsing via :c:func:`PyArg_ParseTuple`; more information on this function is provided below. -The :const:`METH_KEYWORDS` bit may be set in the third field if keyword +The :c:macro:`METH_KEYWORDS` bit may be set in the third field if keyword arguments should be passed to the function. In this case, the C function should accept a third ``PyObject *`` parameter which will be a dictionary of keywords. Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a @@ -354,7 +354,7 @@ The method table must be referenced in the module definition structure:: This structure, in turn, must be passed to the interpreter in the module's initialization function. The initialization function must be named -:c:func:`PyInit_name`, where *name* is the name of the module, and should be the +:c:func:`!PyInit_name`, where *name* is the name of the module, and should be the only non-\ ``static`` item defined in the module file:: PyMODINIT_FUNC @@ -363,12 +363,12 @@ only non-\ ``static`` item defined in the module file:: return PyModule_Create(&spammodule); } -Note that PyMODINIT_FUNC declares the function as ``PyObject *`` return type, +Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, declares any special linkage declarations required by the platform, and for C++ declares the function as ``extern "C"``. -When the Python program imports module :mod:`spam` for the first time, -:c:func:`PyInit_spam` is called. (See below for comments about embedding Python.) +When the Python program imports module :mod:`!spam` for the first time, +:c:func:`!PyInit_spam` is called. (See below for comments about embedding Python.) It calls :c:func:`PyModule_Create`, which returns a module object, and inserts built-in function objects into the newly created module based upon the table (an array of :c:type:`PyMethodDef` structures) found in the module definition. @@ -378,19 +378,20 @@ certain errors, or return ``NULL`` if the module could not be initialized satisfactorily. The init function must return the module object to its caller, so that it then gets inserted into ``sys.modules``. -When embedding Python, the :c:func:`PyInit_spam` function is not called +When embedding Python, the :c:func:`!PyInit_spam` function is not called automatically unless there's an entry in the :c:data:`PyImport_Inittab` table. To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`, optionally followed by an import of the module:: + #define PY_SSIZE_T_CLEAN + #include + int main(int argc, char *argv[]) { - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); - } + PyStatus status; + PyConfig config; + PyConfig_InitPythonConfig(&config); /* Add a built-in module, before Py_Initialize */ if (PyImport_AppendInittab("spam", PyInit_spam) == -1) { @@ -399,11 +400,18 @@ optionally followed by an import of the module:: } /* Pass argv[0] to the Python interpreter */ - Py_SetProgramName(program); + status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]); + if (PyStatus_Exception(status)) { + goto exception; + } /* Initialize the Python interpreter. Required. If this step fails, it will be a fatal error. */ - Py_Initialize(); + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; + } + PyConfig_Clear(&config); /* Optionally import the module; alternatively, import can be deferred until the embedded script @@ -414,10 +422,13 @@ optionally followed by an import of the module:: fprintf(stderr, "Error: could not import module 'spam'\n"); } - ... + // ... use Python C API here ... - PyMem_RawFree(program); return 0; + + exception: + PyConfig_Clear(&config); + Py_ExitStatusException(status); } .. note:: @@ -527,7 +538,7 @@ be part of a module definition:: } This function must be registered with the interpreter using the -:const:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The +:c:macro:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The :c:func:`PyArg_ParseTuple` function and its arguments are documented in section :ref:`parsetuple`. @@ -649,7 +660,7 @@ Note that any Python object references which are provided to the caller are Some example calls:: - #define PY_SSIZE_T_CLEAN /* Make "s#" use Py_ssize_t rather than int. */ + #define PY_SSIZE_T_CLEAN #include :: @@ -724,7 +735,7 @@ Keyword Parameters for Extension Functions The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: int PyArg_ParseTupleAndKeywords(PyObject *arg, PyObject *kwdict, - const char *format, char *kwlist[], ...); + const char *format, char * const *kwlist, ...); The *arg* and *format* parameters are identical to those of the :c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of @@ -745,7 +756,7 @@ it returns false and raises an appropriate exception. Here is an example module which uses keywords, based on an example by Geoff Philbrick (philbrick@hks.com):: - #define PY_SSIZE_T_CLEAN /* Make "s#" use Py_ssize_t rather than int. */ + #define PY_SSIZE_T_CLEAN #include static PyObject * @@ -1030,13 +1041,13 @@ Let's follow the control flow into :c:func:`PyList_SetItem`. The list owns references to all its items, so when item 1 is replaced, it has to dispose of the original item 1. Now let's suppose the original item 1 was an instance of a user-defined class, and let's further suppose that the class defined a -:meth:`__del__` method. If this class instance has a reference count of 1, -disposing of it will call its :meth:`__del__` method. +:meth:`!__del__` method. If this class instance has a reference count of 1, +disposing of it will call its :meth:`!__del__` method. -Since it is written in Python, the :meth:`__del__` method can execute arbitrary +Since it is written in Python, the :meth:`!__del__` method can execute arbitrary Python code. Could it perhaps do something to invalidate the reference to -``item`` in :c:func:`bug`? You bet! Assuming that the list passed into -:c:func:`bug` is accessible to the :meth:`__del__` method, it could execute a +``item`` in :c:func:`!bug`? You bet! Assuming that the list passed into +:c:func:`!bug` is accessible to the :meth:`!__del__` method, it could execute a statement to the effect of ``del list[0]``, and assuming this was the last reference to that object, it would free the memory associated with it, thereby invalidating ``item``. @@ -1057,7 +1068,7 @@ increment the reference count. The correct version of the function reads:: This is a true story. An older version of Python contained variants of this bug and someone spent a considerable amount of time in a C debugger to figure out -why his :meth:`__del__` methods would fail... +why his :meth:`!__del__` methods would fail... The second case of problems with a borrowed reference is a variant involving threads. Normally, multiple threads in the Python interpreter can't get in each @@ -1208,14 +1219,14 @@ file corresponding to the module provides a macro that takes care of importing the module and retrieving its C API pointers; client modules only have to call this macro before accessing the C API. -The exporting module is a modification of the :mod:`spam` module from section -:ref:`extending-simpleexample`. The function :func:`spam.system` does not call +The exporting module is a modification of the :mod:`!spam` module from section +:ref:`extending-simpleexample`. The function :func:`!spam.system` does not call the C library function :c:func:`system` directly, but a function -:c:func:`PySpam_System`, which would of course do something more complicated in +:c:func:`!PySpam_System`, which would of course do something more complicated in reality (such as adding "spam" to every command). This function -:c:func:`PySpam_System` is also exported to other extension modules. +:c:func:`!PySpam_System` is also exported to other extension modules. -The function :c:func:`PySpam_System` is a plain C function, declared +The function :c:func:`!PySpam_System` is a plain C function, declared ``static`` like everything else:: static int @@ -1224,7 +1235,7 @@ The function :c:func:`PySpam_System` is a plain C function, declared return system(command); } -The function :c:func:`spam_system` is modified in a trivial way:: +The function :c:func:`!spam_system` is modified in a trivial way:: static PyObject * spam_system(PyObject *self, PyObject *args) @@ -1268,8 +1279,7 @@ function must take care of initializing the C API pointer array:: /* Create a Capsule containing the API pointer array's address */ c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); - if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) { - Py_XDECREF(c_api_object); + if (PyModule_Add(m, "_C_API", c_api_object) < 0) { Py_DECREF(m); return NULL; } @@ -1278,7 +1288,7 @@ function must take care of initializing the C API pointer array:: } Note that ``PySpam_API`` is declared ``static``; otherwise the pointer -array would disappear when :func:`PyInit_spam` terminates! +array would disappear when :c:func:`!PyInit_spam` terminates! The bulk of the work is in the header file :file:`spammodule.h`, which looks like this:: @@ -1332,8 +1342,8 @@ like this:: #endif /* !defined(Py_SPAMMODULE_H) */ All that a client module must do in order to have access to the function -:c:func:`PySpam_System` is to call the function (or rather macro) -:c:func:`import_spam` in its initialization function:: +:c:func:`!PySpam_System` is to call the function (or rather macro) +:c:func:`!import_spam` in its initialization function:: PyMODINIT_FUNC PyInit_client(void) diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 6852a385f0c63c..7a92b3257c6cd3 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -168,7 +168,7 @@ representation of the instance for which it is called. Here is a simple example:: static PyObject * - newdatatype_repr(newdatatypeobject * obj) + newdatatype_repr(newdatatypeobject *obj) { return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}", obj->obj_UnderlyingDatatypePtr->size); @@ -188,7 +188,7 @@ used instead. Here is a simple example:: static PyObject * - newdatatype_str(newdatatypeobject * obj) + newdatatype_str(newdatatypeobject *obj) { return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}", obj->obj_UnderlyingDatatypePtr->size); @@ -270,7 +270,7 @@ structure:: One entry should be defined for each method provided by the type; no entries are needed for methods inherited from a base type. One additional entry is needed at the end; it is a sentinel that marks the end of the array. The -:attr:`ml_name` field of the sentinel must be ``NULL``. +:c:member:`~PyMethodDef.ml_name` field of the sentinel must be ``NULL``. The second table is used to define attributes which map directly to data stored in the instance. A variety of primitive C types are supported, and access may @@ -286,9 +286,9 @@ be read-only or read-write. The structures in the table are defined as:: For each entry in the table, a :term:`descriptor` will be constructed and added to the type which will be able to extract a value from the instance structure. The -:attr:`type` field should contain a type code like :c:macro:`Py_T_INT` or +:c:member:`~PyMemberDef.type` field should contain a type code like :c:macro:`Py_T_INT` or :c:macro:`Py_T_DOUBLE`; the value will be used to determine how to -convert Python values to and from C values. The :attr:`flags` field is used to +convert Python values to and from C values. The :c:member:`~PyMemberDef.flags` field is used to store flags which control how the attribute can be accessed: you can set it to :c:macro:`Py_READONLY` to prevent Python code from setting it. @@ -296,9 +296,9 @@ An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the -class object, and get the doc string using its :attr:`__doc__` attribute. +class object, and get the doc string using its :attr:`!__doc__` attribute. -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :attr:`name` value +As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. @@ -323,7 +323,7 @@ called, so that if you do need to extend their functionality, you'll understand what needs to be done. The :c:member:`~PyTypeObject.tp_getattr` handler is called when the object requires an attribute -look-up. It is called in the same situations where the :meth:`__getattr__` +look-up. It is called in the same situations where the :meth:`~object.__getattr__` method of a class would be called. Here is an example:: @@ -338,12 +338,12 @@ Here is an example:: PyErr_Format(PyExc_AttributeError, "'%.100s' object has no attribute '%.400s'", - tp->tp_name, name); + Py_TYPE(obj)->tp_name, name); return NULL; } -The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`__setattr__` or -:meth:`__delattr__` method of a class instance would be called. When an +The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`~object.__setattr__` or +:meth:`~object.__delattr__` method of a class instance would be called. When an attribute should be deleted, the third parameter will be ``NULL``. Here is an example that simply raises an exception; if this were really all you wanted, the :c:member:`~PyTypeObject.tp_setattr` handler should be set to ``NULL``. :: @@ -364,7 +364,7 @@ Object Comparison The :c:member:`~PyTypeObject.tp_richcompare` handler is called when comparisons are needed. It is analogous to the :ref:`rich comparison methods `, like -:meth:`__lt__`, and also called by :c:func:`PyObject_RichCompare` and +:meth:`!__lt__`, and also called by :c:func:`PyObject_RichCompare` and :c:func:`PyObject_RichCompareBool`. This function is called with two Python objects and the operator as arguments, @@ -379,7 +379,7 @@ Here is a sample implementation, for a datatype that is considered equal if the size of an internal pointer is equal:: static PyObject * - newdatatype_richcmp(PyObject *obj1, PyObject *obj2, int op) + newdatatype_richcmp(newdatatypeobject *obj1, newdatatypeobject *obj2, int op) { PyObject *result; int c, size1, size2; @@ -478,7 +478,7 @@ This function takes three arguments: Here is a toy ``tp_call`` implementation:: static PyObject * - newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds) + newdatatype_call(newdatatypeobject *obj, PyObject *args, PyObject *kwds) { PyObject *result; const char *arg1; @@ -505,7 +505,7 @@ These functions provide support for the iterator protocol. Both handlers take exactly one parameter, the instance for which they are being called, and return a new reference. In the case of an error, they should set an exception and return ``NULL``. :c:member:`~PyTypeObject.tp_iter` corresponds -to the Python :meth:`__iter__` method, while :c:member:`~PyTypeObject.tp_iternext` +to the Python :meth:`~object.__iter__` method, while :c:member:`~PyTypeObject.tp_iternext` corresponds to the Python :meth:`~iterator.__next__` method. Any :term:`iterable` object must implement the :c:member:`~PyTypeObject.tp_iter` diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index f89934a11f12a8..7eba9759119b3b 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -36,8 +36,8 @@ So, if you want to define a new extension type, you need to create a new type object. This sort of thing can only be explained by example, so here's a minimal, but -complete, module that defines a new type named :class:`Custom` inside a C -extension module :mod:`custom`: +complete, module that defines a new type named :class:`!Custom` inside a C +extension module :mod:`!custom`: .. note:: What we're showing here is the traditional way of defining *static* @@ -45,17 +45,17 @@ extension module :mod:`custom`: allows defining heap-allocated extension types using the :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial. -.. literalinclude:: ../includes/custom.c +.. literalinclude:: ../includes/newtypes/custom.c Now that's quite a bit to take in at once, but hopefully bits will seem familiar from the previous chapter. This file defines three things: -#. What a :class:`Custom` **object** contains: this is the ``CustomObject`` - struct, which is allocated once for each :class:`Custom` instance. -#. How the :class:`Custom` **type** behaves: this is the ``CustomType`` struct, +#. What a :class:`!Custom` **object** contains: this is the ``CustomObject`` + struct, which is allocated once for each :class:`!Custom` instance. +#. How the :class:`!Custom` **type** behaves: this is the ``CustomType`` struct, which defines a set of flags and function pointers that the interpreter inspects when specific operations are requested. -#. How to initialize the :mod:`custom` module: this is the ``PyInit_custom`` +#. How to initialize the :mod:`!custom` module: this is the ``PyInit_custom`` function and the associated ``custommodule`` struct. The first bit is:: @@ -127,8 +127,8 @@ our objects and in some error messages, for example: TypeError: can only concatenate str (not "custom.Custom") to str Note that the name is a dotted name that includes both the module name and the -name of the type within the module. The module in this case is :mod:`custom` and -the type is :class:`Custom`, so we set the type name to :class:`custom.Custom`. +name of the type within the module. The module in this case is :mod:`!custom` and +the type is :class:`!Custom`, so we set the type name to :class:`!custom.Custom`. Using the real dotted import path is important to make your type compatible with the :mod:`pydoc` and :mod:`pickle` modules. :: @@ -136,7 +136,7 @@ with the :mod:`pydoc` and :mod:`pickle` modules. :: .tp_itemsize = 0, This is so that Python knows how much memory to allocate when creating -new :class:`Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is +new :class:`!Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is only used for variable-sized objects and should otherwise be zero. .. note:: @@ -145,13 +145,13 @@ only used for variable-sized objects and should otherwise be zero. :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first in its :attr:`~class.__bases__`, or else it will not be able to call your type's - :meth:`__new__` method without getting an error. You can avoid this problem by + :meth:`~object.__new__` method without getting an error. You can avoid this problem by ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your base type will be :class:`object`, or else you will be adding data members to your base type, and therefore increasing its size. -We set the class flags to :const:`Py_TPFLAGS_DEFAULT`. :: +We set the class flags to :c:macro:`Py_TPFLAGS_DEFAULT`. :: .tp_flags = Py_TPFLAGS_DEFAULT, @@ -164,31 +164,29 @@ We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: .tp_doc = PyDoc_STR("Custom objects"), To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` -handler. This is the equivalent of the Python method :meth:`__new__`, but +handler. This is the equivalent of the Python method :meth:`~object.__new__`, but has to be specified explicitly. In this case, we can just use the default implementation provided by the API function :c:func:`PyType_GenericNew`. :: .tp_new = PyType_GenericNew, Everything else in the file should be familiar, except for some code in -:c:func:`PyInit_custom`:: +:c:func:`!PyInit_custom`:: if (PyType_Ready(&CustomType) < 0) return; -This initializes the :class:`Custom` type, filling in a number of members -to the appropriate default values, including :attr:`ob_type` that we initially +This initializes the :class:`!Custom` type, filling in a number of members +to the appropriate default values, including :c:member:`~PyObject.ob_type` that we initially set to ``NULL``. :: - Py_INCREF(&CustomType); - if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(&CustomType); + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { Py_DECREF(m); return NULL; } This adds the type to the module dictionary. This allows us to create -:class:`Custom` instances by calling the :class:`Custom` class: +:class:`!Custom` instances by calling the :class:`!Custom` class: .. code-block:: pycon @@ -196,50 +194,46 @@ This adds the type to the module dictionary. This allows us to create >>> mycustom = custom.Custom() That's it! All that remains is to build it; put the above code in a file called -:file:`custom.c` and: +:file:`custom.c`, + +.. literalinclude:: ../includes/newtypes/pyproject.toml + +in a file called :file:`pyproject.toml`, and .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[Extension("custom", ["custom.c"])]) + from setuptools import Extension, setup + setup(ext_modules=[Extension("custom", ["custom.c"])]) in a file called :file:`setup.py`; then typing .. code-block:: shell-session - $ python setup.py build + $ python -m pip install . -at a shell should produce a file :file:`custom.so` in a subdirectory; move to -that directory and fire up Python --- you should be able to ``import custom`` and -play around with Custom objects. +in a shell should produce a file :file:`custom.so` in a subdirectory +and install it; now fire up Python --- you should be able to ``import custom`` +and play around with ``Custom`` objects. That wasn't so hard, was it? Of course, the current Custom type is pretty uninteresting. It has no data and doesn't do anything. It can't even be subclassed. -.. note:: - While this documentation showcases the standard :mod:`distutils` module - for building C extensions, it is recommended in real-world use cases to - use the newer and better-maintained ``setuptools`` library. Documentation - on how to do this is out of scope for this document and can be found in - the `Python Packaging User's Guide `_. - Adding data and methods to the Basic example ============================================ Let's extend the basic example to add some data and methods. Let's also make -the type usable as a base class. We'll create a new module, :mod:`custom2` that +the type usable as a base class. We'll create a new module, :mod:`!custom2` that adds these capabilities: -.. literalinclude:: ../includes/custom2.c +.. literalinclude:: ../includes/newtypes/custom2.c This version of the module has a number of changes. -The :class:`Custom` type now has three data attributes in its C struct, +The :class:`!Custom` type now has three data attributes in its C struct, *first*, *last*, and *number*. The *first* and *last* variables are Python strings containing first and last names. The *number* attribute is a C integer. @@ -272,7 +266,7 @@ This method first clears the reference counts of the two Python attributes. ``NULL`` (which might happen here if ``tp_new`` failed midway). It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type (computed by ``Py_TYPE(self)``) to free the object's memory. Note that -the object's type might not be :class:`CustomType`, because the object may +the object's type might not be :class:`!CustomType`, because the object may be an instance of a subclass. .. note:: @@ -311,10 +305,10 @@ and install it in the :c:member:`~PyTypeObject.tp_new` member:: .tp_new = Custom_new, The ``tp_new`` handler is responsible for creating (as opposed to initializing) -objects of the type. It is exposed in Python as the :meth:`__new__` method. +objects of the type. It is exposed in Python as the :meth:`~object.__new__` method. It is not required to define a ``tp_new`` member, and indeed many extension types will simply reuse :c:func:`PyType_GenericNew` as done in the first -version of the ``Custom`` type above. In this case, we use the ``tp_new`` +version of the :class:`!Custom` type above. In this case, we use the ``tp_new`` handler to initialize the ``first`` and ``last`` attributes to non-``NULL`` default values. @@ -345,7 +339,7 @@ result against ``NULL`` before proceeding. .. note:: If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one - that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), + that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`~object.__new__`), you must *not* try to determine what method to call using method resolution order at runtime. Always statically determine what type you are going to call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via @@ -388,14 +382,14 @@ by filling the :c:member:`~PyTypeObject.tp_init` slot. :: .tp_init = (initproc) Custom_init, The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the -:meth:`__init__` method. It is used to initialize an object after it's +:meth:`~object.__init__` method. It is used to initialize an object after it's created. Initializers always accept positional and keyword arguments, and they should return either ``0`` on success or ``-1`` on error. Unlike the ``tp_new`` handler, there is no guarantee that ``tp_init`` is called at all (for example, the :mod:`pickle` module by default -doesn't call :meth:`__init__` on unpickled instances). It can also be -called multiple times. Anyone can call the :meth:`__init__` method on +doesn't call :meth:`~object.__init__` on unpickled instances). It can also be +called multiple times. Anyone can call the :meth:`!__init__` method on our objects. For this reason, we have to be extra careful when assigning the new attribute values. We might be tempted, for example to assign the ``first`` member like this:: @@ -453,7 +447,7 @@ Further, the attributes can be deleted, setting the C pointers to ``NULL``. Eve though we can make sure the members are initialized to non-``NULL`` values, the members can be set to ``NULL`` if the attributes are deleted. -We define a single method, :meth:`Custom.name()`, that outputs the objects name as the +We define a single method, :meth:`!Custom.name()`, that outputs the objects name as the concatenation of the first and last names. :: static PyObject * @@ -470,8 +464,8 @@ concatenation of the first and last names. :: return PyUnicode_FromFormat("%S %S", self->first, self->last); } -The method is implemented as a C function that takes a :class:`Custom` (or -:class:`Custom` subclass) instance as the first argument. Methods always take an +The method is implemented as a C function that takes a :class:`!Custom` (or +:class:`!Custom` subclass) instance as the first argument. Methods always take an instance as the first argument. Methods often take positional and keyword arguments as well, but in this case we don't take any and don't need to accept a positional argument tuple or keyword argument dictionary. This method is @@ -482,8 +476,8 @@ equivalent to the Python method: def name(self): return "%s %s" % (self.first, self.last) -Note that we have to check for the possibility that our :attr:`first` and -:attr:`last` members are ``NULL``. This is because they can be deleted, in which +Note that we have to check for the possibility that our :attr:`!first` and +:attr:`!last` members are ``NULL``. This is because they can be deleted, in which case they are set to ``NULL``. It would be better to prevent deletion of these attributes and to restrict the attribute values to be strings. We'll see how to do that in the next section. @@ -498,7 +492,7 @@ definitions:: {NULL} /* Sentinel */ }; -(note that we used the :const:`METH_NOARGS` flag to indicate that the method +(note that we used the :c:macro:`METH_NOARGS` flag to indicate that the method is expecting no arguments other than *self*) and assign it to the :c:member:`~PyTypeObject.tp_methods` slot:: @@ -508,41 +502,45 @@ and assign it to the :c:member:`~PyTypeObject.tp_methods` slot:: Finally, we'll make our type usable as a base class for subclassing. We've written our methods carefully so far so that they don't make any assumptions about the type of the object being created or used, so all we need to do is -to add the :const:`Py_TPFLAGS_BASETYPE` to our class flag definition:: +to add the :c:macro:`Py_TPFLAGS_BASETYPE` to our class flag definition:: .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, -We rename :c:func:`PyInit_custom` to :c:func:`PyInit_custom2`, update the +We rename :c:func:`!PyInit_custom` to :c:func:`!PyInit_custom2`, update the module name in the :c:type:`PyModuleDef` struct, and update the full class name in the :c:type:`PyTypeObject` struct. -Finally, we update our :file:`setup.py` file to build the new module: +Finally, we update our :file:`setup.py` file to include the new module, .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[ - Extension("custom", ["custom.c"]), - Extension("custom2", ["custom2.c"]), - ]) + from setuptools import Extension, setup + setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + ]) + +and then we re-install so that we can ``import custom2``: + +.. code-block:: shell-session + $ python -m pip install . Providing finer control over data attributes ============================================ -In this section, we'll provide finer control over how the :attr:`first` and -:attr:`last` attributes are set in the :class:`Custom` example. In the previous -version of our module, the instance variables :attr:`first` and :attr:`last` +In this section, we'll provide finer control over how the :attr:`!first` and +:attr:`!last` attributes are set in the :class:`!Custom` example. In the previous +version of our module, the instance variables :attr:`!first` and :attr:`!last` could be set to non-string values or even deleted. We want to make sure that these attributes always contain strings. -.. literalinclude:: ../includes/custom3.c +.. literalinclude:: ../includes/newtypes/custom3.c -To provide greater control, over the :attr:`first` and :attr:`last` attributes, +To provide greater control, over the :attr:`!first` and :attr:`!last` attributes, we'll use custom getter and setter functions. Here are the functions for -getting and setting the :attr:`first` attribute:: +getting and setting the :attr:`!first` attribute:: static PyObject * Custom_getfirst(CustomObject *self, void *closure) @@ -571,13 +569,13 @@ getting and setting the :attr:`first` attribute:: return 0; } -The getter function is passed a :class:`Custom` object and a "closure", which is +The getter function is passed a :class:`!Custom` object and a "closure", which is a void pointer. In this case, the closure is ignored. (The closure supports an advanced usage in which definition data is passed to the getter and setter. This could, for example, be used to allow a single set of getter and setter functions that decide the attribute to get or set based on data in the closure.) -The setter function is passed the :class:`Custom` object, the new value, and the +The setter function is passed the :class:`!Custom` object, the new value, and the closure. The new value may be ``NULL``, in which case the attribute is being deleted. In our setter, we raise an error if the attribute is deleted or if its new value is not a string. @@ -666,11 +664,11 @@ still has a reference from itself. Its reference count doesn't drop to zero. Fortunately, Python's cyclic garbage collector will eventually figure out that the list is garbage and free it. -In the second version of the :class:`Custom` example, we allowed any kind of -object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. +In the second version of the :class:`!Custom` example, we allowed any kind of +object to be stored in the :attr:`!first` or :attr:`!last` attributes [#]_. Besides, in the second and third versions, we allowed subclassing -:class:`Custom`, and subclasses may add arbitrary attributes. For any of -those two reasons, :class:`Custom` objects can participate in cycles: +:class:`!Custom`, and subclasses may add arbitrary attributes. For any of +those two reasons, :class:`!Custom` objects can participate in cycles: .. code-block:: pycon @@ -680,11 +678,11 @@ those two reasons, :class:`Custom` objects can participate in cycles: >>> n = Derived() >>> n.some_attribute = n -To allow a :class:`Custom` instance participating in a reference cycle to -be properly detected and collected by the cyclic GC, our :class:`Custom` type +To allow a :class:`!Custom` instance participating in a reference cycle to +be properly detected and collected by the cyclic GC, our :class:`!Custom` type needs to fill two additional slots and to enable a flag that enables these slots: -.. literalinclude:: ../includes/custom4.c +.. literalinclude:: ../includes/newtypes/custom4.c First, the traversal method lets the cyclic GC know about subobjects that could @@ -708,8 +706,8 @@ participate in cycles:: } For each subobject that can participate in cycles, we need to call the -:c:func:`visit` function, which is passed to the traversal method. The -:c:func:`visit` function takes as arguments the subobject and the extra argument +:c:func:`!visit` function, which is passed to the traversal method. The +:c:func:`!visit` function takes as arguments the subobject and the extra argument *arg* passed to the traversal method. It returns an integer value that must be returned if it is non-zero. @@ -774,7 +772,7 @@ and ``Custom_clear``:: Py_TYPE(self)->tp_free((PyObject *) self); } -Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: +Finally, we add the :c:macro:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, @@ -791,9 +789,9 @@ types. It is easiest to inherit from the built in types, since an extension can easily use the :c:type:`PyTypeObject` it needs. It can be difficult to share these :c:type:`PyTypeObject` structures between extension modules. -In this example we will create a :class:`SubList` type that inherits from the +In this example we will create a :class:`!SubList` type that inherits from the built-in :class:`list` type. The new type will be completely compatible with -regular lists, but will have an additional :meth:`increment` method that +regular lists, but will have an additional :meth:`!increment` method that increases an internal counter: .. code-block:: pycon @@ -808,10 +806,10 @@ increases an internal counter: >>> print(s.increment()) 2 -.. literalinclude:: ../includes/sublist.c +.. literalinclude:: ../includes/newtypes/sublist.c -As you can see, the source code closely resembles the :class:`Custom` examples in +As you can see, the source code closely resembles the :class:`!Custom` examples in previous sections. We will break down the main differences between them. :: typedef struct { @@ -823,7 +821,7 @@ The primary difference for derived type objects is that the base type's object structure must be the first value. The base type will already include the :c:func:`PyObject_HEAD` at the beginning of its structure. -When a Python object is a :class:`SubList` instance, its ``PyObject *`` pointer +When a Python object is a :class:`!SubList` instance, its ``PyObject *`` pointer can be safely cast to both ``PyListObject *`` and ``SubListObject *``:: static int @@ -835,7 +833,7 @@ can be safely cast to both ``PyListObject *`` and ``SubListObject *``:: return 0; } -We see above how to call through to the :attr:`__init__` method of the base +We see above how to call through to the :meth:`~object.__init__` method of the base type. This pattern is important when writing a type with custom @@ -862,9 +860,7 @@ function:: if (m == NULL) return NULL; - Py_INCREF(&SubListType); - if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(&SubListType); + if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { Py_DECREF(m); return NULL; } @@ -879,7 +875,7 @@ slot with :c:func:`PyType_GenericNew` -- the allocation function from the base type will be inherited. After that, calling :c:func:`PyType_Ready` and adding the type object to the -module is the same as with the basic :class:`Custom` examples. +module is the same as with the basic :class:`!Custom` examples. .. rubric:: Footnotes diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index 1129b0968bc4e6..e366d6cb9f79e3 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -132,4 +132,4 @@ modules (including Python) to be able to see your identifiers, you have to say Developer Studio will throw in a lot of import libraries that you do not really need, adding about 100K to your executable. To get rid of them, use the Project Settings dialog, Link tab, to specify *ignore default libraries*. Add the -correct :file:`msvcrtxx.lib` to the list of libraries. +correct :file:`msvcrt{xx}.lib` to the list of libraries. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 11d01374dc1e79..ae02c443e5938b 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -584,9 +584,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`list.append` method is expected to add new elements +program. For example, the :meth:`!list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`list.append` implementation will actually do this correctly, but it's +your :meth:`!list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index bc3080f60ee237..2a8b976925d042 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -81,13 +81,13 @@ How do I extract C values from a Python object? That depends on the object's type. If it's a tuple, :c:func:`PyTuple_Size` returns its length and :c:func:`PyTuple_GetItem` returns the item at a specified -index. Lists have similar functions, :c:func:`PyListSize` and +index. Lists have similar functions, :c:func:`PyList_Size` and :c:func:`PyList_GetItem`. For bytes, :c:func:`PyBytes_Size` returns its length and :c:func:`PyBytes_AsStringAndSize` provides a pointer to its value and its length. Note that Python bytes objects may contain null bytes so C's -:c:func:`strlen` should not be used. +:c:func:`!strlen` should not be used. To test the type of an object, first make sure it isn't ``NULL``, and then use :c:func:`PyBytes_Check`, :c:func:`PyTuple_Check`, :c:func:`PyList_Check`, etc. diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index a9b2622e02ef3b..8727332594bda6 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -135,7 +135,7 @@ Python versions are numbered "A.B.C" or "A.B": See :pep:`6` for more information about bugfix releases. -Not all releases are bugfix releases. In the run-up to a new major release, a +Not all releases are bugfix releases. In the run-up to a new feature release, a series of development releases are made, denoted as alpha, beta, or release candidate. Alphas are early releases in which interfaces aren't yet finalized; it's not unexpected to see an interface change between two alpha releases. @@ -297,9 +297,9 @@ How stable is Python? Very stable. New, stable releases have been coming out roughly every 6 to 18 months since 1991, and this seems likely to continue. As of version 3.9, -Python will have a major new release every 12 months (:pep:`602`). +Python will have a new feature release every 12 months (:pep:`602`). -The developers issue "bugfix" releases of older versions, so the stability of +The developers issue bugfix releases of older versions, so the stability of existing releases gradually improves. Bugfix releases, indicated by a third component of the version number (e.g. 3.5.3, 3.6.2), are managed for stability; only fixes for known problems are included in a bugfix release, and it's diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 023ffdf0db510a..cfa60feceb31b7 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -43,18 +43,11 @@ applications, the applications will not be truly stand-alone, as the application will still need the Tcl and Tk libraries. One solution is to ship the application with the Tcl and Tk libraries, and point -to them at run-time using the :envvar:`TCL_LIBRARY` and :envvar:`TK_LIBRARY` +to them at run-time using the :envvar:`!TCL_LIBRARY` and :envvar:`!TK_LIBRARY` environment variables. -To get truly stand-alone applications, the Tcl scripts that form the library -have to be integrated into the application as well. One tool supporting that is -SAM (stand-alone modules), which is part of the Tix distribution -(https://tix.sourceforge.net/). - -Build Tix with SAM enabled, perform the appropriate call to -:c:func:`Tclsam_init`, etc. inside Python's -:file:`Modules/tkappinit.c`, and link with libtclsam and libtksam (you -might include the Tix libraries as well). +Various third-party freeze libraries such as py2exe and cx_Freeze have +handling for Tkinter applications built-in. Can I have Tk events handled while waiting for I/O? @@ -62,7 +55,7 @@ Can I have Tk events handled while waiting for I/O? On platforms other than Windows, yes, and you don't even need threads! But you'll have to restructure your I/O -code a bit. Tk has the equivalent of Xt's :c:func:`XtAddInput()` call, which allows you +code a bit. Tk has the equivalent of Xt's :c:func:`!XtAddInput` call, which allows you to register a callback function which will be called from the Tk mainloop when I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. @@ -70,8 +63,9 @@ I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. I can't get key bindings to work in Tkinter: why? ------------------------------------------------- -An often-heard complaint is that event handlers bound to events with the -:meth:`bind` method don't get handled even when the appropriate key is pressed. +An often-heard complaint is that event handlers :ref:`bound ` +to events with the :meth:`!bind` method +don't get handled even when the appropriate key is pressed. The most common cause is that the widget to which the binding applies doesn't have "keyboard focus". Check out the Tk documentation for the focus command. diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 597caaa778e1c8..476a43d9c288f1 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -111,7 +111,7 @@ Is there an equivalent to C's onexit() in Python? ------------------------------------------------- The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`onexit`. +:c:func:`!onexit`. Why don't my signal handlers work? @@ -397,7 +397,7 @@ These aren't:: D[x] = D[x] + 1 Operations that replace other objects may invoke those other objects' -:meth:`__del__` method when their reference count reaches zero, and that can +:meth:`~object.__del__` method when their reference count reaches zero, and that can affect things. This is especially true for the mass updates to dictionaries and lists. When in doubt, use a mutex! @@ -566,7 +566,7 @@ use ``p.read(n)``. Note on a bug in popen2: unless your program calls ``wait()`` or ``waitpid()``, finished child processes are never removed, and eventually calls to popen2 will fail because of a limit on the number of child - processes. Calling :func:`os.waitpid` with the :data:`os.WNOHANG` option can + processes. Calling :func:`os.waitpid` with the :const:`os.WNOHANG` option can prevent this; a good place to insert such a call would be before calling ``popen2`` again. @@ -669,41 +669,6 @@ and client-side web systems. A summary of available frameworks is maintained by Paul Boddie at https://wiki.python.org/moin/WebProgramming\ . -Cameron Laird maintains a useful set of pages about Python web technologies at -https://web.archive.org/web/20210224183619/http://phaseit.net/claird/comp.lang.python/web_python. - - -How can I mimic CGI form submission (METHOD=POST)? --------------------------------------------------- - -I would like to retrieve web pages that are the result of POSTing a form. Is -there existing code that would let me do this easily? - -Yes. Here's a simple example that uses :mod:`urllib.request`:: - - #!/usr/local/bin/python - - import urllib.request - - # build the query string - qs = "First=Josephine&MI=Q&Last=Public" - - # connect and send the server a path - req = urllib.request.urlopen('http://www.some-server.out-there' - '/cgi-bin/some-cgi-script', data=qs) - with req: - msg, hdrs = req.read(), req.info() - -Note that in general for percent-encoded POST operations, query strings must be -quoted using :func:`urllib.parse.urlencode`. For example, to send -``name=Guy Steele, Jr.``:: - - >>> import urllib.parse - >>> urllib.parse.urlencode({'name': 'Guy Steele, Jr.'}) - 'name=Guy+Steele%2C+Jr.' - -.. seealso:: :ref:`urllib-howto` for extensive examples. - What module should I use to help with generating HTML? ------------------------------------------------------ @@ -765,14 +730,17 @@ The :mod:`select` module is commonly used to help with asynchronous I/O on sockets. To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the :meth:`socket.connect`, you will either connect immediately +mode. Then when you do the :meth:`~socket.socket.connect`, +you will either connect immediately (unlikely) or get an exception that contains the error number as ``.errno``. ``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't finished yet. Different OSes will return different values, so you're going to have to check what's returned on your system. -You can use the :meth:`socket.connect_ex` method to avoid creating an exception. It will -just return the errno value. To poll, you can call :meth:`socket.connect_ex` again later +You can use the :meth:`~socket.socket.connect_ex` method +to avoid creating an exception. +It will just return the errno value. +To poll, you can call :meth:`~socket.socket.connect_ex` again later -- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this socket to :meth:`select.select` to check if it's writable. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ab5618db84f77e..0a88c5f6384f2b 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -454,7 +454,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`~list.append`, the content of the mutable object has +After the call to :meth:`!append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -924,12 +924,12 @@ module:: 'Hello, there!' >>> import array - >>> a = array.array('u', s) + >>> a = array.array('w', s) >>> print(a) - array('u', 'Hello, world') + array('w', 'Hello, world') >>> a[0] = 'y' >>> print(a) - array('u', 'yello, world') + array('w', 'yello, world') >>> a.tounicode() 'yello, world' @@ -1397,7 +1397,7 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list +and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list and returning the list. That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`!list.extend`:: @@ -1903,7 +1903,7 @@ identity tests. This prevents the code from being confused by objects such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of -:meth:`collections.abc.Sequence.__contains__`:: +:meth:`!collections.abc.Sequence.__contains__`:: def __contains__(self, value): for v in self: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 53e8cdcae1cd66..601443d5aade94 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -22,15 +22,6 @@ Glossary * The :const:`Ellipsis` built-in constant. - 2to3 - A tool that tries to convert Python 2.x code to Python 3.x code by - handling most of the incompatibilities which can be detected by parsing the - source and traversing the parse tree. - - 2to3 is available in the standard library as :mod:`lib2to3`; a standalone - entry point is provided as :file:`Tools/scripts/2to3`. See - :ref:`2to3-reference`. - abstract base class Abstract base classes complement :term:`duck-typing` by providing a way to define interfaces when other techniques like @@ -92,8 +83,8 @@ Glossary asynchronous context manager An object which controls the environment seen in an - :keyword:`async with` statement by defining :meth:`__aenter__` and - :meth:`__aexit__` methods. Introduced by :pep:`492`. + :keyword:`async with` statement by defining :meth:`~object.__aenter__` and + :meth:`~object.__aexit__` methods. Introduced by :pep:`492`. asynchronous generator A function which returns an :term:`asynchronous generator iterator`. It @@ -113,26 +104,26 @@ Glossary An object created by a :term:`asynchronous generator` function. This is an :term:`asynchronous iterator` which when called using the - :meth:`__anext__` method returns an awaitable object which will execute + :meth:`~object.__anext__` method returns an awaitable object which will execute the body of the asynchronous generator function until the next :keyword:`yield` expression. Each :keyword:`yield` temporarily suspends processing, remembering the location execution state (including local variables and pending try-statements). When the *asynchronous generator iterator* effectively - resumes with another awaitable returned by :meth:`__anext__`, it + resumes with another awaitable returned by :meth:`~object.__anext__`, it picks up where it left off. See :pep:`492` and :pep:`525`. asynchronous iterable An object, that can be used in an :keyword:`async for` statement. Must return an :term:`asynchronous iterator` from its - :meth:`__aiter__` method. Introduced by :pep:`492`. + :meth:`~object.__aiter__` method. Introduced by :pep:`492`. asynchronous iterator - An object that implements the :meth:`__aiter__` and :meth:`__anext__` - methods. ``__anext__`` must return an :term:`awaitable` object. + An object that implements the :meth:`~object.__aiter__` and :meth:`~object.__anext__` + methods. :meth:`~object.__anext__` must return an :term:`awaitable` object. :keyword:`async for` resolves the awaitables returned by an asynchronous - iterator's :meth:`__anext__` method until it raises a + iterator's :meth:`~object.__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. attribute @@ -149,7 +140,7 @@ Glossary awaitable An object that can be used in an :keyword:`await` expression. Can be - a :term:`coroutine` or an object with an :meth:`__await__` method. + a :term:`coroutine` or an object with an :meth:`~object.__await__` method. See also :pep:`492`. BDFL @@ -160,16 +151,17 @@ Glossary A :term:`file object` able to read and write :term:`bytes-like objects `. Examples of binary files are files opened in binary mode (``'rb'``, - ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, - :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and - :class:`gzip.GzipFile`. + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer `, + :data:`sys.stdout.buffer `, and instances of + :class:`io.BytesIO` and :class:`gzip.GzipFile`. See also :term:`text file` for a file object able to read and write :class:`str` objects. borrowed reference - In Python's C API, a borrowed reference is a reference to an object. - It does not modify the object reference count. It becomes a dangling + In Python's C API, a borrowed reference is a reference to an object, + where the code using the object does not own the reference. + It becomes a dangling pointer if the object is destroyed. For example, a garbage collection can remove the last :term:`strong reference` to the object and so destroy it. @@ -247,7 +239,7 @@ Glossary context manager An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`__enter__` and :meth:`__exit__` methods. + statement by defining :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. See :pep:`343`. context variable @@ -312,8 +304,9 @@ Glossary :ref:`class definitions ` for more about decorators. descriptor - Any object which defines the methods :meth:`__get__`, :meth:`__set__`, or - :meth:`__delete__`. When a class attribute is a descriptor, its special + Any object which defines the methods :meth:`~object.__get__`, + :meth:`~object.__set__`, or :meth:`~object.__delete__`. + When a class attribute is a descriptor, its special binding behavior is triggered upon attribute lookup. Normally, using *a.b* to get, set or delete an attribute looks up the object named *b* in the class dictionary for *a*, but if *b* is a descriptor, the respective @@ -327,7 +320,8 @@ Glossary dictionary An associative array, where arbitrary keys are mapped to values. The - keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. + keys can be any object with :meth:`~object.__hash__` and + :meth:`~object.__eq__` methods. Called a hash in Perl. dictionary comprehension @@ -391,7 +385,7 @@ Glossary file object An object exposing a file-oriented API (with methods such as - :meth:`read()` or :meth:`write()`) to an underlying resource. Depending + :meth:`!read` or :meth:`!write`) to an underlying resource. Depending on the way it was created, a file object can mediate access to a real on-disk file or to another type of storage or communication device (for example standard input/output, in-memory buffers, sockets, pipes, @@ -510,7 +504,7 @@ Glossary .. index:: single: generator expression generator expression - An expression that returns an iterator. It looks like a normal expression + An :term:`expression` that returns an :term:`iterator`. It looks like a normal expression followed by a :keyword:`!for` clause defining a loop variable, range, and an optional :keyword:`!if` clause. The combined expression generates values for an enclosing function:: @@ -567,8 +561,9 @@ Glossary hashable An object is *hashable* if it has a hash value which never changes during - its lifetime (it needs a :meth:`__hash__` method), and can be compared to - other objects (it needs an :meth:`__eq__` method). Hashable objects which + its lifetime (it needs a :meth:`~object.__hash__` method), and can be + compared to other objects (it needs an :meth:`~object.__eq__` method). + Hashable objects which compare equal must have the same hash value. Hashability makes an object usable as a dictionary key and a set member, @@ -587,6 +582,16 @@ Glossary :ref:`idle` is a basic editor and interpreter environment which ships with the standard distribution of Python. + immortal + If an object is immortal, its reference count is never modified, and + therefore it is never deallocated. + + Built-in strings and singletons are immortal objects. For example, + :const:`True` and :const:`None` singletons are immmortal. + + See `PEP 683 – Immortal Objects, Using a Fixed Refcount + `_ for more information. + immutable An object with a fixed value. Immutable objects include numbers, strings and tuples. Such an object cannot be altered. A new object has to @@ -644,7 +649,8 @@ Glossary iterables include all sequence types (such as :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define - with an :meth:`__iter__` method or with a :meth:`__getitem__` method + with an :meth:`~iterator.__iter__` method or with a + :meth:`~object.__getitem__` method that implements :term:`sequence` semantics. Iterables can be @@ -653,7 +659,7 @@ Glossary as an argument to the built-in function :func:`iter`, it returns an iterator for the object. This iterator is good for one pass over the set of values. When using iterables, it is usually not necessary to call - :func:`iter` or deal with iterator objects yourself. The ``for`` + :func:`iter` or deal with iterator objects yourself. The :keyword:`for` statement does that automatically for you, creating a temporary unnamed variable to hold the iterator for the duration of the loop. See also :term:`iterator`, :term:`sequence`, and :term:`generator`. @@ -664,8 +670,8 @@ Glossary :func:`next`) return successive items in the stream. When no more data are available a :exc:`StopIteration` exception is raised instead. At this point, the iterator object is exhausted and any further calls to its - :meth:`__next__` method just raise :exc:`StopIteration` again. Iterators - are required to have an :meth:`__iter__` method that returns the iterator + :meth:`!__next__` method just raise :exc:`StopIteration` again. Iterators + are required to have an :meth:`~iterator.__iter__` method that returns the iterator object itself so every iterator is also iterable and may be used in most places where other iterables are accepted. One notable exception is code which attempts multiple iteration passes. A container object (such as a @@ -679,7 +685,7 @@ Glossary .. impl-detail:: CPython does not consistently apply the requirement that an iterator - define :meth:`__iter__`. + define :meth:`~iterator.__iter__`. key function A key function or collation function is a callable that returns a value @@ -873,7 +879,8 @@ Glossary Old name for the flavor of classes now used for all class objects. In earlier Python versions, only new-style classes could use Python's newer, versatile features like :attr:`~object.__slots__`, descriptors, - properties, :meth:`__getattribute__`, class methods, and static methods. + properties, :meth:`~object.__getattribute__`, class methods, and static + methods. object Any data with state (attributes or value) and defined behavior @@ -953,7 +960,7 @@ Glossary finders implement. path entry hook - A callable on the :data:`sys.path_hook` list which returns a :term:`path + A callable on the :data:`sys.path_hooks` list which returns a :term:`path entry finder` if it knows how to find modules on a specific :term:`path entry`. @@ -1063,7 +1070,9 @@ Glossary reference count The number of references to an object. When the reference count of an - object drops to zero, it is deallocated. Reference counting is + object drops to zero, it is deallocated. Some objects are + :term:`immortal` and have reference counts that are never modified, and + therefore the objects are never deallocated. Reference counting is generally not visible to Python code, but it is a key element of the :term:`CPython` implementation. Programmers can call the :func:`sys.getrefcount` function to return the @@ -1084,19 +1093,19 @@ Glossary sequence An :term:`iterable` which supports efficient element access using integer - indices via the :meth:`__getitem__` special method and defines a - :meth:`__len__` method that returns the length of the sequence. + indices via the :meth:`~object.__getitem__` special method and defines a + :meth:`~object.__len__` method that returns the length of the sequence. Some built-in sequence types are :class:`list`, :class:`str`, :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also - supports :meth:`__getitem__` and :meth:`__len__`, but is considered a + supports :meth:`~object.__getitem__` and :meth:`!__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary :term:`immutable` keys rather than integers. The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just - :meth:`__getitem__` and :meth:`__len__`, adding :meth:`count`, - :meth:`index`, :meth:`__contains__`, and - :meth:`__reversed__`. Types that implement this expanded + :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding + :meth:`count`, :meth:`index`, :meth:`~object.__contains__`, and + :meth:`~object.__reversed__`. Types that implement this expanded interface can be registered explicitly using :func:`~abc.ABCMeta.register`. @@ -1116,6 +1125,21 @@ Glossary when several are given, such as in ``variable_name[1:3:5]``. The bracket (subscript) notation uses :class:`slice` objects internally. + soft deprecated + A soft deprecation can be used when using an API which should no longer + be used to write new code, but it remains safe to continue using it in + existing code. The API remains documented and tested, but will not be + developed further (no enhancement). + + The main difference between a "soft" and a (regular) "hard" deprecation + is that the soft deprecation does not imply scheduling the removal of the + deprecated API. + + Another difference is that a soft deprecation does not issue a warning. + + See `PEP 387: Soft Deprecation + `_. + special method .. index:: pair: special; method @@ -1129,10 +1153,17 @@ Glossary an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. + static type checker + An external tool that reads Python code and analyzes it, looking for + issues such as incorrect types. See also :term:`type hints ` + and the :mod:`typing` module. + strong reference In Python's C API, a strong reference is a reference to an object - which increments the object's reference count when it is created and - decrements the object's reference count when it is deleted. + which is owned by the code holding the reference. The strong + reference is taken by calling :c:func:`Py_INCREF` when the + reference is created and released with :c:func:`Py_DECREF` + when the reference is deleted. The :c:func:`Py_NewRef` function can be used to create a strong reference to an object. Usually, the :c:func:`Py_DECREF` function must be called on @@ -1203,8 +1234,8 @@ Glossary attribute, or a function parameter or return value. Type hints are optional and are not enforced by Python but - they are useful to static type analysis tools, and aid IDEs with code - completion and refactoring. + they are useful to :term:`static type checkers `. + They can also aid IDEs with code completion and refactoring. Type hints of global variables, class attributes, and functions, but not local variables, can be accessed using diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 472069032d6509..1134686c947d66 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -32,201 +32,201 @@ Annotations Best Practices Accessing The Annotations Dict Of An Object In Python 3.10 And Newer ==================================================================== - Python 3.10 adds a new function to the standard library: - :func:`inspect.get_annotations`. In Python versions 3.10 - and newer, calling this function is the best practice for - accessing the annotations dict of any object that supports - annotations. This function can also "un-stringize" - stringized annotations for you. - - If for some reason :func:`inspect.get_annotations` isn't - viable for your use case, you may access the - ``__annotations__`` data member manually. Best practice - for this changed in Python 3.10 as well: as of Python 3.10, - ``o.__annotations__`` is guaranteed to *always* work - on Python functions, classes, and modules. If you're - certain the object you're examining is one of these three - *specific* objects, you may simply use ``o.__annotations__`` - to get at the object's annotations dict. - - However, other types of callables--for example, - callables created by :func:`functools.partial`--may - not have an ``__annotations__`` attribute defined. When - accessing the ``__annotations__`` of a possibly unknown - object, best practice in Python versions 3.10 and - newer is to call :func:`getattr` with three arguments, - for example ``getattr(o, '__annotations__', None)``. - - Before Python 3.10, accessing ``__annotations__`` on a class that - defines no annotations but that has a parent class with - annotations would return the parent's ``__annotations__``. - In Python 3.10 and newer, the child class's annotations - will be an empty dict instead. +Python 3.10 adds a new function to the standard library: +:func:`inspect.get_annotations`. In Python versions 3.10 +and newer, calling this function is the best practice for +accessing the annotations dict of any object that supports +annotations. This function can also "un-stringize" +stringized annotations for you. + +If for some reason :func:`inspect.get_annotations` isn't +viable for your use case, you may access the +``__annotations__`` data member manually. Best practice +for this changed in Python 3.10 as well: as of Python 3.10, +``o.__annotations__`` is guaranteed to *always* work +on Python functions, classes, and modules. If you're +certain the object you're examining is one of these three +*specific* objects, you may simply use ``o.__annotations__`` +to get at the object's annotations dict. + +However, other types of callables--for example, +callables created by :func:`functools.partial`--may +not have an ``__annotations__`` attribute defined. When +accessing the ``__annotations__`` of a possibly unknown +object, best practice in Python versions 3.10 and +newer is to call :func:`getattr` with three arguments, +for example ``getattr(o, '__annotations__', None)``. + +Before Python 3.10, accessing ``__annotations__`` on a class that +defines no annotations but that has a parent class with +annotations would return the parent's ``__annotations__``. +In Python 3.10 and newer, the child class's annotations +will be an empty dict instead. Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== - In Python 3.9 and older, accessing the annotations dict - of an object is much more complicated than in newer versions. - The problem is a design flaw in these older versions of Python, - specifically to do with class annotations. +In Python 3.9 and older, accessing the annotations dict +of an object is much more complicated than in newer versions. +The problem is a design flaw in these older versions of Python, +specifically to do with class annotations. - Best practice for accessing the annotations dict of other - objects--functions, other callables, and modules--is the same - as best practice for 3.10, assuming you aren't calling - :func:`inspect.get_annotations`: you should use three-argument - :func:`getattr` to access the object's ``__annotations__`` - attribute. +Best practice for accessing the annotations dict of other +objects--functions, other callables, and modules--is the same +as best practice for 3.10, assuming you aren't calling +:func:`inspect.get_annotations`: you should use three-argument +:func:`getattr` to access the object's ``__annotations__`` +attribute. - Unfortunately, this isn't best practice for classes. The problem - is that, since ``__annotations__`` is optional on classes, and - because classes can inherit attributes from their base classes, - accessing the ``__annotations__`` attribute of a class may - inadvertently return the annotations dict of a *base class.* - As an example:: +Unfortunately, this isn't best practice for classes. The problem +is that, since ``__annotations__`` is optional on classes, and +because classes can inherit attributes from their base classes, +accessing the ``__annotations__`` attribute of a class may +inadvertently return the annotations dict of a *base class.* +As an example:: - class Base: - a: int = 3 - b: str = 'abc' + class Base: + a: int = 3 + b: str = 'abc' - class Derived(Base): - pass + class Derived(Base): + pass - print(Derived.__annotations__) + print(Derived.__annotations__) - This will print the annotations dict from ``Base``, not - ``Derived``. +This will print the annotations dict from ``Base``, not +``Derived``. - Your code will have to have a separate code path if the object - you're examining is a class (``isinstance(o, type)``). - In that case, best practice relies on an implementation detail - of Python 3.9 and before: if a class has annotations defined, - they are stored in the class's ``__dict__`` dictionary. Since - the class may or may not have annotations defined, best practice - is to call the ``get`` method on the class dict. +Your code will have to have a separate code path if the object +you're examining is a class (``isinstance(o, type)``). +In that case, best practice relies on an implementation detail +of Python 3.9 and before: if a class has annotations defined, +they are stored in the class's ``__dict__`` dictionary. Since +the class may or may not have annotations defined, best practice +is to call the ``get`` method on the class dict. - To put it all together, here is some sample code that safely - accesses the ``__annotations__`` attribute on an arbitrary - object in Python 3.9 and before:: +To put it all together, here is some sample code that safely +accesses the ``__annotations__`` attribute on an arbitrary +object in Python 3.9 and before:: - if isinstance(o, type): - ann = o.__dict__.get('__annotations__', None) - else: - ann = getattr(o, '__annotations__', None) + if isinstance(o, type): + ann = o.__dict__.get('__annotations__', None) + else: + ann = getattr(o, '__annotations__', None) - After running this code, ``ann`` should be either a - dictionary or ``None``. You're encouraged to double-check - the type of ``ann`` using :func:`isinstance` before further - examination. +After running this code, ``ann`` should be either a +dictionary or ``None``. You're encouraged to double-check +the type of ``ann`` using :func:`isinstance` before further +examination. - Note that some exotic or malformed type objects may not have - a ``__dict__`` attribute, so for extra safety you may also wish - to use :func:`getattr` to access ``__dict__``. +Note that some exotic or malformed type objects may not have +a ``__dict__`` attribute, so for extra safety you may also wish +to use :func:`getattr` to access ``__dict__``. Manually Un-Stringizing Stringized Annotations ============================================== - In situations where some annotations may be "stringized", - and you wish to evaluate those strings to produce the - Python values they represent, it really is best to - call :func:`inspect.get_annotations` to do this work - for you. - - If you're using Python 3.9 or older, or if for some reason - you can't use :func:`inspect.get_annotations`, you'll need - to duplicate its logic. You're encouraged to examine the - implementation of :func:`inspect.get_annotations` in the - current Python version and follow a similar approach. - - In a nutshell, if you wish to evaluate a stringized annotation - on an arbitrary object ``o``: - - * If ``o`` is a module, use ``o.__dict__`` as the - ``globals`` when calling :func:`eval`. - * If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` - as the ``globals``, and ``dict(vars(o))`` as the ``locals``, - when calling :func:`eval`. - * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively - unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as - appropriate, until you have found the root unwrapped function. - * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. - - However, not all string values used as annotations can - be successfully turned into Python values by :func:`eval`. - String values could theoretically contain any valid string, - and in practice there are valid use cases for type hints that - require annotating with string values that specifically - *can't* be evaluated. For example: - - * :pep:`604` union types using ``|``, before support for this - was added to Python 3.10. - * Definitions that aren't needed at runtime, only imported - when :const:`typing.TYPE_CHECKING` is true. - - If :func:`eval` attempts to evaluate such values, it will - fail and raise an exception. So, when designing a library - API that works with annotations, it's recommended to only - attempt to evaluate string values when explicitly requested - to by the caller. +In situations where some annotations may be "stringized", +and you wish to evaluate those strings to produce the +Python values they represent, it really is best to +call :func:`inspect.get_annotations` to do this work +for you. + +If you're using Python 3.9 or older, or if for some reason +you can't use :func:`inspect.get_annotations`, you'll need +to duplicate its logic. You're encouraged to examine the +implementation of :func:`inspect.get_annotations` in the +current Python version and follow a similar approach. + +In a nutshell, if you wish to evaluate a stringized annotation +on an arbitrary object ``o``: + +* If ``o`` is a module, use ``o.__dict__`` as the + ``globals`` when calling :func:`eval`. +* If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` + as the ``globals``, and ``dict(vars(o))`` as the ``locals``, + when calling :func:`eval`. +* If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, + :func:`functools.wraps`, or :func:`functools.partial`, iteratively + unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as + appropriate, until you have found the root unwrapped function. +* If ``o`` is a callable (but not a class), use + ``o.__globals__`` as the globals when calling :func:`eval`. + +However, not all string values used as annotations can +be successfully turned into Python values by :func:`eval`. +String values could theoretically contain any valid string, +and in practice there are valid use cases for type hints that +require annotating with string values that specifically +*can't* be evaluated. For example: + +* :pep:`604` union types using ``|``, before support for this + was added to Python 3.10. +* Definitions that aren't needed at runtime, only imported + when :const:`typing.TYPE_CHECKING` is true. + +If :func:`eval` attempts to evaluate such values, it will +fail and raise an exception. So, when designing a library +API that works with annotations, it's recommended to only +attempt to evaluate string values when explicitly requested +to by the caller. Best Practices For ``__annotations__`` In Any Python Version ============================================================ - * You should avoid assigning to the ``__annotations__`` member - of objects directly. Let Python manage setting ``__annotations__``. +* You should avoid assigning to the ``__annotations__`` member + of objects directly. Let Python manage setting ``__annotations__``. - * If you do assign directly to the ``__annotations__`` member - of an object, you should always set it to a ``dict`` object. +* If you do assign directly to the ``__annotations__`` member + of an object, you should always set it to a ``dict`` object. - * If you directly access the ``__annotations__`` member - of an object, you should ensure that it's a - dictionary before attempting to examine its contents. +* If you directly access the ``__annotations__`` member + of an object, you should ensure that it's a + dictionary before attempting to examine its contents. - * You should avoid modifying ``__annotations__`` dicts. +* You should avoid modifying ``__annotations__`` dicts. - * You should avoid deleting the ``__annotations__`` attribute - of an object. +* You should avoid deleting the ``__annotations__`` attribute + of an object. ``__annotations__`` Quirks ========================== - In all versions of Python 3, function - objects lazy-create an annotations dict if no annotations - are defined on that object. You can delete the ``__annotations__`` - attribute using ``del fn.__annotations__``, but if you then - access ``fn.__annotations__`` the object will create a new empty dict - that it will store and return as its annotations. Deleting the - annotations on a function before it has lazily created its annotations - dict will throw an ``AttributeError``; using ``del fn.__annotations__`` - twice in a row is guaranteed to always throw an ``AttributeError``. - - Everything in the above paragraph also applies to class and module - objects in Python 3.10 and newer. - - In all versions of Python 3, you can set ``__annotations__`` - on a function object to ``None``. However, subsequently - accessing the annotations on that object using ``fn.__annotations__`` - will lazy-create an empty dictionary as per the first paragraph of - this section. This is *not* true of modules and classes, in any Python - version; those objects permit setting ``__annotations__`` to any - Python value, and will retain whatever value is set. - - If Python stringizes your annotations for you - (using ``from __future__ import annotations``), and you - specify a string as an annotation, the string will - itself be quoted. In effect the annotation is quoted - *twice.* For example:: - - from __future__ import annotations - def foo(a: "str"): pass - - print(foo.__annotations__) - - This prints ``{'a': "'str'"}``. This shouldn't really be considered - a "quirk"; it's mentioned here simply because it might be surprising. +In all versions of Python 3, function +objects lazy-create an annotations dict if no annotations +are defined on that object. You can delete the ``__annotations__`` +attribute using ``del fn.__annotations__``, but if you then +access ``fn.__annotations__`` the object will create a new empty dict +that it will store and return as its annotations. Deleting the +annotations on a function before it has lazily created its annotations +dict will throw an ``AttributeError``; using ``del fn.__annotations__`` +twice in a row is guaranteed to always throw an ``AttributeError``. + +Everything in the above paragraph also applies to class and module +objects in Python 3.10 and newer. + +In all versions of Python 3, you can set ``__annotations__`` +on a function object to ``None``. However, subsequently +accessing the annotations on that object using ``fn.__annotations__`` +will lazy-create an empty dictionary as per the first paragraph of +this section. This is *not* true of modules and classes, in any Python +version; those objects permit setting ``__annotations__`` to any +Python value, and will retain whatever value is set. + +If Python stringizes your annotations for you +(using ``from __future__ import annotations``), and you +specify a string as an annotation, the string will +itself be quoted. In effect the annotation is quoted +*twice.* For example:: + + from __future__ import annotations + def foo(a: "str"): pass + + print(foo.__annotations__) + +This prints ``{'a': "'str'"}``. This shouldn't really be considered +a "quirk"; it's mentioned here simply because it might be surprising. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 52e98fa9620194..ae5bab90bf8131 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -788,6 +788,59 @@ but not both at the same time: -q, --quiet +How to translate the argparse output +==================================== + +The output of the :mod:`argparse` module such as its help text and error +messages are all made translatable using the :mod:`gettext` module. This +allows applications to easily localize messages produced by +:mod:`argparse`. See also :ref:`i18n-howto`. + +For instance, in this :mod:`argparse` output: + +.. code-block:: shell-session + + $ python prog.py --help + usage: prog.py [-h] [-v | -q] x y + + calculate X to the power of Y + + positional arguments: + x the base + y the exponent + + options: + -h, --help show this help message and exit + -v, --verbose + -q, --quiet + +The strings ``usage:``, ``positional arguments:``, ``options:`` and +``show this help message and exit`` are all translatable. + +In order to translate these strings, they must first be extracted +into a ``.po`` file. For example, using `Babel `__, +run this command: + +.. code-block:: shell-session + + $ pybabel extract -o messages.po /usr/lib/python3.12/argparse.py + +This command will extract all translatable strings from the :mod:`argparse` +module and output them into a file named ``messages.po``. This command assumes +that your Python installation is in ``/usr/lib``. + +You can find out the location of the :mod:`argparse` module on your system +using this script:: + + import argparse + print(argparse.__file__) + +Once the messages in the ``.po`` file are translated and the translations are +installed using :mod:`gettext`, :mod:`argparse` will be able to display the +translated messages. + +To translate your own strings in the :mod:`argparse` output, use :mod:`gettext`. + Conclusion ========== diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 4620b4617e3450..060977246149cf 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1,1822 +1,14 @@ -.. highlight:: c +:orphan: -.. _howto-clinic: +.. This page is retained solely for existing links to /howto/clinic.html. + Direct readers to the devguide. ********************** Argument Clinic How-To ********************** -:author: Larry Hastings +.. note:: -.. topic:: Abstract - - Argument Clinic is a preprocessor for CPython C files. - Its purpose is to automate all the boilerplate involved - with writing argument parsing code for "builtins". - This document shows you how to convert your first C - function to work with Argument Clinic, and then introduces - some advanced topics on Argument Clinic usage. - - Currently Argument Clinic is considered internal-only - for CPython. Its use is not supported for files outside - CPython, and no guarantees are made regarding backwards - compatibility for future versions. In other words: if you - maintain an external C extension for CPython, you're welcome - to experiment with Argument Clinic in your own code. But the - version of Argument Clinic that ships with the next version - of CPython *could* be totally incompatible and break all your code. - -The Goals Of Argument Clinic -============================ - -Argument Clinic's primary goal -is to take over responsibility for all argument parsing code -inside CPython. This means that, when you convert a function -to work with Argument Clinic, that function should no longer -do any of its own argument parsing—the code generated by -Argument Clinic should be a "black box" to you, where CPython -calls in at the top, and your code gets called at the bottom, -with ``PyObject *args`` (and maybe ``PyObject *kwargs``) -magically converted into the C variables and types you need. - -In order for Argument Clinic to accomplish its primary goal, -it must be easy to use. Currently, working with CPython's -argument parsing library is a chore, requiring maintaining -redundant information in a surprising number of places. -When you use Argument Clinic, you don't have to repeat yourself. - -Obviously, no one would want to use Argument Clinic unless -it's solving their problem—and without creating new problems of -its own. -So it's paramount that Argument Clinic generate correct code. -It'd be nice if the code was faster, too, but at the very least -it should not introduce a major speed regression. (Eventually Argument -Clinic *should* make a major speedup possible—we could -rewrite its code generator to produce tailor-made argument -parsing code, rather than calling the general-purpose CPython -argument parsing library. That would make for the fastest -argument parsing possible!) - -Additionally, Argument Clinic must be flexible enough to -work with any approach to argument parsing. Python has -some functions with some very strange parsing behaviors; -Argument Clinic's goal is to support all of them. - -Finally, the original motivation for Argument Clinic was -to provide introspection "signatures" for CPython builtins. -It used to be, the introspection query functions would throw -an exception if you passed in a builtin. With Argument -Clinic, that's a thing of the past! - -One idea you should keep in mind, as you work with -Argument Clinic: the more information you give it, the -better job it'll be able to do. -Argument Clinic is admittedly relatively simple right -now. But as it evolves it will get more sophisticated, -and it should be able to do many interesting and smart -things with all the information you give it. - - -Basic Concepts And Usage -======================== - -Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``. -If you run that script, specifying a C file as an argument: - -.. code-block:: shell-session - - $ python Tools/clinic/clinic.py foo.c - -Argument Clinic will scan over the file looking for lines that -look exactly like this: - -.. code-block:: none - - /*[clinic input] - -When it finds one, it reads everything up to a line that looks -exactly like this: - -.. code-block:: none - - [clinic start generated code]*/ - -Everything in between these two lines is input for Argument Clinic. -All of these lines, including the beginning and ending comment -lines, are collectively called an Argument Clinic "block". - -When Argument Clinic parses one of these blocks, it -generates output. This output is rewritten into the C file -immediately after the block, followed by a comment containing a checksum. -The Argument Clinic block now looks like this: - -.. code-block:: none - - /*[clinic input] - ... clinic input goes here ... - [clinic start generated code]*/ - ... clinic output goes here ... - /*[clinic end generated code: checksum=...]*/ - -If you run Argument Clinic on the same file a second time, Argument Clinic -will discard the old output and write out the new output with a fresh checksum -line. However, if the input hasn't changed, the output won't change either. - -You should never modify the output portion of an Argument Clinic block. Instead, -change the input until it produces the output you want. (That's the purpose of the -checksum—to detect if someone changed the output, as these edits would be lost -the next time Argument Clinic writes out fresh output.) - -For the sake of clarity, here's the terminology we'll use with Argument Clinic: - -* The first line of the comment (``/*[clinic input]``) is the *start line*. -* The last line of the initial comment (``[clinic start generated code]*/``) is the *end line*. -* The last line (``/*[clinic end generated code: checksum=...]*/``) is the *checksum line*. -* In between the start line and the end line is the *input*. -* In between the end line and the checksum line is the *output*. -* All the text collectively, from the start line to the checksum line inclusively, - is the *block*. (A block that hasn't been successfully processed by Argument - Clinic yet doesn't have output or a checksum line, but it's still considered - a block.) - - -Converting Your First Function -============================== - -The best way to get a sense of how Argument Clinic works is to -convert a function to work with it. Here, then, are the bare -minimum steps you'd need to follow to convert a function to -work with Argument Clinic. Note that for code you plan to -check in to CPython, you really should take the conversion farther, -using some of the advanced concepts you'll see later on in -the document (like "return converters" and "self converters"). -But we'll keep it simple for this walkthrough so you can learn. - -Let's dive in! - -0. Make sure you're working with a freshly updated checkout - of the CPython trunk. - -1. Find a Python builtin that calls either :c:func:`PyArg_ParseTuple` - or :c:func:`PyArg_ParseTupleAndKeywords`, and hasn't been converted - to work with Argument Clinic yet. - For my example I'm using ``_pickle.Pickler.dump()``. - -2. If the call to the ``PyArg_Parse`` function uses any of the - following format units: - - .. code-block:: none - - O& - O! - es - es# - et - et# - - or if it has multiple calls to :c:func:`PyArg_ParseTuple`, - you should choose a different function. Argument Clinic *does* - support all of these scenarios. But these are advanced - topics—let's do something simpler for your first function. - - Also, if the function has multiple calls to :c:func:`PyArg_ParseTuple` - or :c:func:`PyArg_ParseTupleAndKeywords` where it supports different - types for the same argument, or if the function uses something besides - PyArg_Parse functions to parse its arguments, it probably - isn't suitable for conversion to Argument Clinic. Argument Clinic - doesn't support generic functions or polymorphic parameters. - -3. Add the following boilerplate above the function, creating our block:: - - /*[clinic input] - [clinic start generated code]*/ - -4. Cut the docstring and paste it in between the ``[clinic]`` lines, - removing all the junk that makes it a properly quoted C string. - When you're done you should have just the text, based at the left - margin, with no line wider than 80 characters. - (Argument Clinic will preserve indents inside the docstring.) - - If the old docstring had a first line that looked like a function - signature, throw that line away. (The docstring doesn't need it - anymore—when you use ``help()`` on your builtin in the future, - the first line will be built automatically based on the function's - signature.) - - Sample:: - - /*[clinic input] - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -5. If your docstring doesn't have a "summary" line, Argument Clinic will - complain. So let's make sure it has one. The "summary" line should - be a paragraph consisting of a single 80-column line - at the beginning of the docstring. - - (Our example docstring consists solely of a summary line, so the sample - code doesn't have to change for this step.) - -6. Above the docstring, enter the name of the function, followed - by a blank line. This should be the Python name of the function, - and should be the full dotted path - to the function—it should start with the name of the module, - include any sub-modules, and if the function is a method on - a class it should include the class name too. - - Sample:: - - /*[clinic input] - _pickle.Pickler.dump - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -7. If this is the first time that module or class has been used with Argument - Clinic in this C file, - you must declare the module and/or class. Proper Argument Clinic hygiene - prefers declaring these in a separate block somewhere near the - top of the C file, in the same way that include files and statics go at - the top. (In our sample code we'll just show the two blocks next to - each other.) - - The name of the class and module should be the same as the one - seen by Python. Check the name defined in the :c:type:`PyModuleDef` - or :c:type:`PyTypeObject` as appropriate. - - When you declare a class, you must also specify two aspects of its type - in C: the type declaration you'd use for a pointer to an instance of - this class, and a pointer to the :c:type:`PyTypeObject` for this class. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - - - -8. Declare each of the parameters to the function. Each parameter - should get its own line. All the parameter lines should be - indented from the function name and the docstring. - - The general form of these parameter lines is as follows: - - .. code-block:: none - - name_of_parameter: converter - - If the parameter has a default value, add that after the - converter: - - .. code-block:: none - - name_of_parameter: converter = default_value - - Argument Clinic's support for "default values" is quite sophisticated; - please see :ref:`the section below on default values ` - for more information. - - Add a blank line below the parameters. - - What's a "converter"? It establishes both the type - of the variable used in C, and the method to convert the Python - value into a C value at runtime. - For now you're going to use what's called a "legacy converter"—a - convenience syntax intended to make porting old code into Argument - Clinic easier. - - For each parameter, copy the "format unit" for that - parameter from the ``PyArg_Parse()`` format argument and - specify *that* as its converter, as a quoted - string. ("format unit" is the formal name for the one-to-three - character substring of the ``format`` parameter that tells - the argument parsing function what the type of the variable - is and how to convert it. For more on format units please - see :ref:`arg-parsing`.) - - For multicharacter format units like ``z#``, use the - entire two-or-three character string. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -9. If your function has ``|`` in the format string, meaning some - parameters have default values, you can ignore it. Argument - Clinic infers which parameters are optional based on whether - or not they have default values. - - If your function has ``$`` in the format string, meaning it - takes keyword-only arguments, specify ``*`` on a line by - itself before the first keyword-only argument, indented the - same as the parameter lines. - - (``_pickle.Pickler.dump`` has neither, so our sample is unchanged.) - - -10. If the existing C function calls :c:func:`PyArg_ParseTuple` - (as opposed to :c:func:`PyArg_ParseTupleAndKeywords`), then all its - arguments are positional-only. - - To mark all parameters as positional-only in Argument Clinic, - add a ``/`` on a line by itself after the last parameter, - indented the same as the parameter lines. - - Currently this is all-or-nothing; either all parameters are - positional-only, or none of them are. (In the future Argument - Clinic may relax this restriction.) - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -11. It's helpful to write a per-parameter docstring for each parameter. - But per-parameter docstrings are optional; you can skip this step - if you prefer. - - Here's how to add a per-parameter docstring. The first line - of the per-parameter docstring must be indented further than the - parameter definition. The left margin of this first line establishes - the left margin for the whole per-parameter docstring; all the text - you write will be outdented by this amount. You can write as much - text as you like, across multiple lines if you wish. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -12. Save and close the file, then run ``Tools/clinic/clinic.py`` on - it. With luck everything worked---your block now has output, and - a ``.c.h`` file has been generated! Reopen the file in your - text editor to see:: - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - static PyObject * - _pickle_Pickler_dump(PicklerObject *self, PyObject *obj) - /*[clinic end generated code: output=87ecad1261e02ac7 input=552eb1c0f52260d9]*/ - - Obviously, if Argument Clinic didn't produce any output, it's because - it found an error in your input. Keep fixing your errors and retrying - until Argument Clinic processes your file without complaint. - - For readability, most of the glue code has been generated to a ``.c.h`` - file. You'll need to include that in your original ``.c`` file, - typically right after the clinic module block:: - - #include "clinic/_pickle.c.h" - -13. Double-check that the argument-parsing code Argument Clinic generated - looks basically the same as the existing code. - - First, ensure both places use the same argument-parsing function. - The existing code must call either - :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_ParseTupleAndKeywords`; - ensure that the code generated by Argument Clinic calls the - *exact* same function. - - Second, the format string passed in to :c:func:`PyArg_ParseTuple` or - :c:func:`PyArg_ParseTupleAndKeywords` should be *exactly* the same - as the hand-written one in the existing function, up to the colon - or semi-colon. - - (Argument Clinic always generates its format strings - with a ``:`` followed by the name of the function. If the - existing code's format string ends with ``;``, to provide - usage help, this change is harmless—don't worry about it.) - - Third, for parameters whose format units require two arguments - (like a length variable, or an encoding string, or a pointer - to a conversion function), ensure that the second argument is - *exactly* the same between the two invocations. - - Fourth, inside the output portion of the block you'll find a preprocessor - macro defining the appropriate static :c:type:`PyMethodDef` structure for - this builtin:: - - #define __PICKLE_PICKLER_DUMP_METHODDEF \ - {"dump", (PyCFunction)__pickle_Pickler_dump, METH_O, __pickle_Pickler_dump__doc__}, - - This static structure should be *exactly* the same as the existing static - :c:type:`PyMethodDef` structure for this builtin. - - If any of these items differ in *any way*, - adjust your Argument Clinic function specification and rerun - ``Tools/clinic/clinic.py`` until they *are* the same. - - -14. Notice that the last line of its output is the declaration - of your "impl" function. This is where the builtin's implementation goes. - Delete the existing prototype of the function you're modifying, but leave - the opening curly brace. Now delete its argument parsing code and the - declarations of all the variables it dumps the arguments into. - Notice how the Python arguments are now arguments to this impl function; - if the implementation used different names for these variables, fix it. - - Let's reiterate, just because it's kind of weird. Your code should now - look like this:: - - static return_type - your_function_impl(...) - /*[clinic end generated code: checksum=...]*/ - { - ... - - Argument Clinic generated the checksum line and the function prototype just - above it. You should write the opening (and closing) curly braces for the - function, and the implementation inside. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - PyDoc_STRVAR(__pickle_Pickler_dump__doc__, - "Write a pickled representation of obj to the open file.\n" - "\n" - ... - static PyObject * - _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj) - /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/ - { - /* Check whether the Pickler was initialized correctly (issue3664). - Developers often forget to call __init__() in their subclasses, which - would trigger a segfault without this check. */ - if (self->write == NULL) { - PyErr_Format(PicklingError, - "Pickler.__init__() was not called by %s.__init__()", - Py_TYPE(self)->tp_name); - return NULL; - } - - if (_Pickler_ClearBuffer(self) < 0) - return NULL; - - ... - -15. Remember the macro with the :c:type:`PyMethodDef` structure for this - function? Find the existing :c:type:`PyMethodDef` structure for this - function and replace it with a reference to the macro. (If the builtin - is at module scope, this will probably be very near the end of the file; - if the builtin is a class method, this will probably be below but relatively - near to the implementation.) - - Note that the body of the macro contains a trailing comma. So when you - replace the existing static :c:type:`PyMethodDef` structure with the macro, - *don't* add a comma to the end. - - Sample:: - - static struct PyMethodDef Pickler_methods[] = { - __PICKLE_PICKLER_DUMP_METHODDEF - __PICKLE_PICKLER_CLEAR_MEMO_METHODDEF - {NULL, NULL} /* sentinel */ - }; - - -16. Argument Clinic may generate new instances of ``_Py_ID``. For example:: - - &_Py_ID(new_unique_py_id) - - If it does, you'll have to run ``Tools/scripts/generate_global_objects.py`` - to regenerate the list of precompiled identifiers at this point. - - -17. Compile, then run the relevant portions of the regression-test suite. - This change should not introduce any new compile-time warnings or errors, - and there should be no externally visible change to Python's behavior. - - Well, except for one difference: ``inspect.signature()`` run on your function - should now provide a valid signature! - - Congratulations, you've ported your first function to work with Argument Clinic! - -Advanced Topics -=============== - -Now that you've had some experience working with Argument Clinic, it's time -for some advanced topics. - - -Symbolic default values ------------------------ - -The default value you provide for a parameter can't be any arbitrary -expression. Currently the following are explicitly supported: - -* Numeric constants (integer and float) -* String constants -* ``True``, ``False``, and ``None`` -* Simple symbolic constants like ``sys.maxsize``, which must - start with the name of the module - -(In the future, this may need to get even more elaborate, -to allow full expressions like ``CONSTANT - 1``.) - - -Renaming the C functions and variables generated by Argument Clinic -------------------------------------------------------------------- - -Argument Clinic automatically names the functions it generates for you. -Occasionally this may cause a problem, if the generated name collides with -the name of an existing C function. There's an easy solution: override the names -used for the C functions. Just add the keyword ``"as"`` -to your function declaration line, followed by the function name you wish to use. -Argument Clinic will use that function name for the base (generated) function, -then add ``"_impl"`` to the end and use that for the name of the impl function. - -For example, if we wanted to rename the C function names generated for -``pickle.Pickler.dump``, it'd look like this:: - - /*[clinic input] - pickle.Pickler.dump as pickler_dumper - - ... - -The base function would now be named ``pickler_dumper()``, -and the impl function would now be named ``pickler_dumper_impl()``. - - -Similarly, you may have a problem where you want to give a parameter -a specific Python name, but that name may be inconvenient in C. Argument -Clinic allows you to give a parameter different names in Python and in C, -using the same ``"as"`` syntax:: - - /*[clinic input] - pickle.Pickler.dump - - obj: object - file as file_obj: object - protocol: object = NULL - * - fix_imports: bool = True - -Here, the name used in Python (in the signature and the ``keywords`` -array) would be ``file``, but the C variable would be named ``file_obj``. - -You can use this to rename the ``self`` parameter too! - - -Converting functions using PyArg_UnpackTuple --------------------------------------------- - -To convert a function parsing its arguments with :c:func:`PyArg_UnpackTuple`, -simply write out all the arguments, specifying each as an ``object``. You -may specify the ``type`` argument to cast the type as appropriate. All -arguments should be marked positional-only (add a ``/`` on a line by itself -after the last argument). - -Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this -will change soon. - -Optional Groups ---------------- - -Some legacy functions have a tricky approach to parsing their arguments: -they count the number of positional arguments, then use a ``switch`` statement -to call one of several different :c:func:`PyArg_ParseTuple` calls depending on -how many positional arguments there are. (These functions cannot accept -keyword-only arguments.) This approach was used to simulate optional -arguments back before :c:func:`PyArg_ParseTupleAndKeywords` was created. - -While functions using this approach can often be converted to -use :c:func:`PyArg_ParseTupleAndKeywords`, optional arguments, and default values, -it's not always possible. Some of these legacy functions have -behaviors :c:func:`PyArg_ParseTupleAndKeywords` doesn't directly support. -The most obvious example is the builtin function ``range()``, which has -an optional argument on the *left* side of its required argument! -Another example is ``curses.window.addch()``, which has a group of two -arguments that must always be specified together. (The arguments are -called ``x`` and ``y``; if you call the function passing in ``x``, -you must also pass in ``y``—and if you don't pass in ``x`` you may not -pass in ``y`` either.) - -In any case, the goal of Argument Clinic is to support argument parsing -for all existing CPython builtins without changing their semantics. -Therefore Argument Clinic supports -this alternate approach to parsing, using what are called *optional groups*. -Optional groups are groups of arguments that must all be passed in together. -They can be to the left or the right of the required arguments. They -can *only* be used with positional-only parameters. - -.. note:: Optional groups are *only* intended for use when converting - functions that make multiple calls to :c:func:`PyArg_ParseTuple`! - Functions that use *any* other approach for parsing arguments - should *almost never* be converted to Argument Clinic using - optional groups. Functions using optional groups currently - cannot have accurate signatures in Python, because Python just - doesn't understand the concept. Please avoid using optional - groups wherever possible. - -To specify an optional group, add a ``[`` on a line by itself before -the parameters you wish to group together, and a ``]`` on a line by itself -after these parameters. As an example, here's how ``curses.window.addch`` -uses optional groups to make the first two parameters and the last -parameter optional:: - - /*[clinic input] - - curses.window.addch - - [ - x: int - X-coordinate. - y: int - Y-coordinate. - ] - - ch: object - Character to add. - - [ - attr: long - Attributes for the character. - ] - / - - ... - - -Notes: - -* For every optional group, one additional parameter will be passed into the - impl function representing the group. The parameter will be an int named - ``group_{direction}_{number}``, - where ``{direction}`` is either ``right`` or ``left`` depending on whether the group - is before or after the required parameters, and ``{number}`` is a monotonically - increasing number (starting at 1) indicating how far away the group is from - the required parameters. When the impl is called, this parameter will be set - to zero if this group was unused, and set to non-zero if this group was used. - (By used or unused, I mean whether or not the parameters received arguments - in this invocation.) - -* If there are no required arguments, the optional groups will behave - as if they're to the right of the required arguments. - -* In the case of ambiguity, the argument parsing code - favors parameters on the left (before the required parameters). - -* Optional groups can only contain positional-only parameters. - -* Optional groups are *only* intended for legacy code. Please do not - use optional groups for new code. - - -Using real Argument Clinic converters, instead of "legacy converters" ---------------------------------------------------------------------- - -To save time, and to minimize how much you need to learn -to achieve your first port to Argument Clinic, the walkthrough above tells -you to use "legacy converters". "Legacy converters" are a convenience, -designed explicitly to make porting existing code to Argument Clinic -easier. And to be clear, their use is acceptable when porting code for -Python 3.4. - -However, in the long term we probably want all our blocks to -use Argument Clinic's real syntax for converters. Why? A couple -reasons: - -* The proper converters are far easier to read and clearer in their intent. -* There are some format units that are unsupported as "legacy converters", - because they require arguments, and the legacy converter syntax doesn't - support specifying arguments. -* In the future we may have a new argument parsing library that isn't - restricted to what :c:func:`PyArg_ParseTuple` supports; this flexibility - won't be available to parameters using legacy converters. - -Therefore, if you don't mind a little extra effort, please use the normal -converters instead of legacy converters. - -In a nutshell, the syntax for Argument Clinic (non-legacy) converters -looks like a Python function call. However, if there are no explicit -arguments to the function (all functions take their default values), -you may omit the parentheses. Thus ``bool`` and ``bool()`` are exactly -the same converters. - -All arguments to Argument Clinic converters are keyword-only. -All Argument Clinic converters accept the following arguments: - - ``c_default`` - The default value for this parameter when defined in C. - Specifically, this will be the initializer for the variable declared - in the "parse function". See :ref:`the section on default values ` - for how to use this. - Specified as a string. - - ``annotation`` - The annotation value for this parameter. Not currently supported, - because :pep:`8` mandates that the Python library may not use - annotations. - - ``unused`` - Wrap the argument with :c:macro:`Py_UNUSED` in the impl function signature. - -In addition, some converters accept additional arguments. Here is a list -of these arguments, along with their meanings: - - ``accept`` - A set of Python types (and possibly pseudo-types); - this restricts the allowable Python argument to values of these types. - (This is not a general-purpose facility; as a rule it only supports - specific lists of types as shown in the legacy converter table.) - - To accept ``None``, add ``NoneType`` to this set. - - ``bitwise`` - Only supported for unsigned integers. The native integer value of this - Python argument will be written to the parameter without any range checking, - even for negative values. - - ``converter`` - Only supported by the ``object`` converter. Specifies the name of a - :ref:`C "converter function" ` - to use to convert this object to a native type. - - ``encoding`` - Only supported for strings. Specifies the encoding to use when converting - this string from a Python str (Unicode) value into a C ``char *`` value. - - - ``subclass_of`` - Only supported for the ``object`` converter. Requires that the Python - value be a subclass of a Python type, as expressed in C. - - ``type`` - Only supported for the ``object`` and ``self`` converters. Specifies - the C type that will be used to declare the variable. Default value is - ``"PyObject *"``. - - ``zeroes`` - Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are - permitted inside the value. The length of the string will be passed in - to the impl function, just after the string parameter, as a parameter named - ``_length``. - -Please note, not every possible combination of arguments will work. -Usually these arguments are implemented by specific ``PyArg_ParseTuple`` -*format units*, with specific behavior. For example, currently you cannot -call ``unsigned_short`` without also specifying ``bitwise=True``. -Although it's perfectly reasonable to think this would work, these semantics don't -map to any existing format unit. So Argument Clinic doesn't support it. (Or, at -least, not yet.) - -Below is a table showing the mapping of legacy converters into real -Argument Clinic converters. On the left is the legacy converter, -on the right is the text you'd replace it with. - -========= ================================================================================= -``'B'`` ``unsigned_char(bitwise=True)`` -``'b'`` ``unsigned_char`` -``'c'`` ``char`` -``'C'`` ``int(accept={str})`` -``'d'`` ``double`` -``'D'`` ``Py_complex`` -``'es'`` ``str(encoding='name_of_encoding')`` -``'es#'`` ``str(encoding='name_of_encoding', zeroes=True)`` -``'et'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str})`` -``'et#'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str}, zeroes=True)`` -``'f'`` ``float`` -``'h'`` ``short`` -``'H'`` ``unsigned_short(bitwise=True)`` -``'i'`` ``int`` -``'I'`` ``unsigned_int(bitwise=True)`` -``'k'`` ``unsigned_long(bitwise=True)`` -``'K'`` ``unsigned_long_long(bitwise=True)`` -``'l'`` ``long`` -``'L'`` ``long long`` -``'n'`` ``Py_ssize_t`` -``'O'`` ``object`` -``'O!'`` ``object(subclass_of='&PySomething_Type')`` -``'O&'`` ``object(converter='name_of_c_function')`` -``'p'`` ``bool`` -``'S'`` ``PyBytesObject`` -``'s'`` ``str`` -``'s#'`` ``str(zeroes=True)`` -``'s*'`` ``Py_buffer(accept={buffer, str})`` -``'U'`` ``unicode`` -``'u'`` ``wchar_t`` -``'u#'`` ``wchar_t(zeroes=True)`` -``'w*'`` ``Py_buffer(accept={rwbuffer})`` -``'Y'`` ``PyByteArrayObject`` -``'y'`` ``str(accept={bytes})`` -``'y#'`` ``str(accept={robuffer}, zeroes=True)`` -``'y*'`` ``Py_buffer`` -``'Z'`` ``wchar_t(accept={str, NoneType})`` -``'Z#'`` ``wchar_t(accept={str, NoneType}, zeroes=True)`` -``'z'`` ``str(accept={str, NoneType})`` -``'z#'`` ``str(accept={str, NoneType}, zeroes=True)`` -``'z*'`` ``Py_buffer(accept={buffer, str, NoneType})`` -========= ================================================================================= - -As an example, here's our sample ``pickle.Pickler.dump`` using the proper -converter:: - - /*[clinic input] - pickle.Pickler.dump - - obj: object - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -One advantage of real converters is that they're more flexible than legacy -converters. For example, the ``unsigned_int`` converter (and all the -``unsigned_`` converters) can be specified without ``bitwise=True``. Their -default behavior performs range checking on the value, and they won't accept -negative numbers. You just can't do that with a legacy converter! - -Argument Clinic will show you all the converters it has -available. For each converter it'll show you all the parameters -it accepts, along with the default value for each parameter. -Just run ``Tools/clinic/clinic.py --converters`` to see the full list. - -Py_buffer ---------- - -When using the ``Py_buffer`` converter -(or the ``'s*'``, ``'w*'``, ``'*y'``, or ``'z*'`` legacy converters), -you *must* not call :c:func:`PyBuffer_Release` on the provided buffer. -Argument Clinic generates code that does it for you (in the parsing function). - - - -Advanced converters -------------------- - -Remember those format units you skipped for your first -time because they were advanced? Here's how to handle those too. - -The trick is, all those format units take arguments—either -conversion functions, or types, or strings specifying an encoding. -(But "legacy converters" don't support arguments. That's why we -skipped them for your first function.) The argument you specified -to the format unit is now an argument to the converter; this -argument is either ``converter`` (for ``O&``), ``subclass_of`` (for ``O!``), -or ``encoding`` (for all the format units that start with ``e``). - -When using ``subclass_of``, you may also want to use the other -custom argument for ``object()``: ``type``, which lets you set the type -actually used for the parameter. For example, if you want to ensure -that the object is a subclass of ``PyUnicode_Type``, you probably want -to use the converter ``object(type='PyUnicodeObject *', subclass_of='&PyUnicode_Type')``. - -One possible problem with using Argument Clinic: it takes away some possible -flexibility for the format units starting with ``e``. When writing a -``PyArg_Parse`` call by hand, you could theoretically decide at runtime what -encoding string to pass in to :c:func:`PyArg_ParseTuple`. But now this string must -be hard-coded at Argument-Clinic-preprocessing-time. This limitation is deliberate; -it made supporting this format unit much easier, and may allow for future optimizations. -This restriction doesn't seem unreasonable; CPython itself always passes in static -hard-coded encoding strings for parameters whose format units start with ``e``. - - -.. _default_values: - -Parameter default values ------------------------- - -Default values for parameters can be any of a number of values. -At their simplest, they can be string, int, or float literals: - -.. code-block:: none - - foo: str = "abc" - bar: int = 123 - bat: float = 45.6 - -They can also use any of Python's built-in constants: - -.. code-block:: none - - yep: bool = True - nope: bool = False - nada: object = None - -There's also special support for a default value of ``NULL``, and -for simple expressions, documented in the following sections. - - -The ``NULL`` default value --------------------------- - -For string and object parameters, you can set them to ``None`` to indicate -that there's no default. However, that means the C variable will be -initialized to ``Py_None``. For convenience's sakes, there's a special -value called ``NULL`` for just this reason: from Python's perspective it -behaves like a default value of ``None``, but the C variable is initialized -with ``NULL``. - -Expressions specified as default values ---------------------------------------- - -The default value for a parameter can be more than just a literal value. -It can be an entire expression, using math operators and looking up attributes -on objects. However, this support isn't exactly simple, because of some -non-obvious semantics. - -Consider the following example: - -.. code-block:: none - - foo: Py_ssize_t = sys.maxsize - 1 - -``sys.maxsize`` can have different values on different platforms. Therefore -Argument Clinic can't simply evaluate that expression locally and hard-code it -in C. So it stores the default in such a way that it will get evaluated at -runtime, when the user asks for the function's signature. - -What namespace is available when the expression is evaluated? It's evaluated -in the context of the module the builtin came from. So, if your module has an -attribute called "``max_widgets``", you may simply use it: - -.. code-block:: none - - foo: Py_ssize_t = max_widgets - -If the symbol isn't found in the current module, it fails over to looking in -``sys.modules``. That's how it can find ``sys.maxsize`` for example. (Since you -don't know in advance what modules the user will load into their interpreter, -it's best to restrict yourself to modules that are preloaded by Python itself.) - -Evaluating default values only at runtime means Argument Clinic can't compute -the correct equivalent C default value. So you need to tell it explicitly. -When you use an expression, you must also specify the equivalent expression -in C, using the ``c_default`` parameter to the converter: - -.. code-block:: none - - foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1 - -Another complication: Argument Clinic can't know in advance whether or not the -expression you supply is valid. It parses it to make sure it looks legal, but -it can't *actually* know. You must be very careful when using expressions to -specify values that are guaranteed to be valid at runtime! - -Finally, because expressions must be representable as static C values, there -are many restrictions on legal expressions. Here's a list of Python features -you're not permitted to use: - -* Function calls. -* Inline if statements (``3 if foo else 5``). -* Automatic sequence unpacking (``*[1, 2, 3]``). -* List/set/dict comprehensions and generator expressions. -* Tuple/list/set/dict literals. - - - -Using a return converter ------------------------- - -By default, the impl function Argument Clinic generates for you returns -:c:type:`PyObject * `. -But your C function often computes some C type, -then converts it into the :c:type:`!PyObject *` -at the last moment. Argument Clinic handles converting your inputs from Python types -into native C types—why not have it convert your return value from a native C type -into a Python type too? - -That's what a "return converter" does. It changes your impl function to return -some C type, then adds code to the generated (non-impl) function to handle converting -that value into the appropriate :c:type:`!PyObject *`. - -The syntax for return converters is similar to that of parameter converters. -You specify the return converter like it was a return annotation on the -function itself, using ``->`` notation. - -For example: - -.. code-block:: c - - /*[clinic input] - add -> int - - a: int - b: int - / - - [clinic start generated code]*/ - -Return converters behave much the same as parameter converters; -they take arguments, the arguments are all keyword-only, and if you're not changing -any of the default arguments you can omit the parentheses. - -(If you use both ``"as"`` *and* a return converter for your function, -the ``"as"`` should come before the return converter.) - -There's one additional complication when using return converters: how do you -indicate an error has occurred? Normally, a function returns a valid (non-``NULL``) -pointer for success, and ``NULL`` for failure. But if you use an integer return converter, -all integers are valid. How can Argument Clinic detect an error? Its solution: each return -converter implicitly looks for a special value that indicates an error. If you return -that value, and an error has been set (``PyErr_Occurred()`` returns a true -value), then the generated code will propagate the error. Otherwise it will -encode the value you return like normal. - -Currently Argument Clinic supports only a few return converters: - -.. code-block:: none - - bool - double - float - int - long - Py_ssize_t - size_t - unsigned int - unsigned long - -None of these take parameters. -For all of these, return ``-1`` to indicate error. - -To see all the return converters Argument Clinic supports, along with -their parameters (if any), -just run ``Tools/clinic/clinic.py --converters`` for the full list. - - -Cloning existing functions --------------------------- - -If you have a number of functions that look similar, you may be able to -use Clinic's "clone" feature. When you clone an existing function, -you reuse: - -* its parameters, including - - * their names, - - * their converters, with all parameters, - - * their default values, - - * their per-parameter docstrings, - - * their *kind* (whether they're positional only, - positional or keyword, or keyword only), and - -* its return converter. - -The only thing not copied from the original function is its docstring; -the syntax allows you to specify a new docstring. - -Here's the syntax for cloning a function:: - - /*[clinic input] - module.class.new_function [as c_basename] = module.class.existing_function - - Docstring for new_function goes here. - [clinic start generated code]*/ - -(The functions can be in different modules or classes. I wrote -``module.class`` in the sample just to illustrate that you must -use the full path to *both* functions.) - -Sorry, there's no syntax for partially cloning a function, or cloning a function -then modifying it. Cloning is an all-or nothing proposition. - -Also, the function you are cloning from must have been previously defined -in the current file. - -Calling Python code -------------------- - -The rest of the advanced topics require you to write Python code -which lives inside your C file and modifies Argument Clinic's -runtime state. This is simple: you simply define a Python block. - -A Python block uses different delimiter lines than an Argument -Clinic function block. It looks like this:: - - /*[python input] - # python code goes here - [python start generated code]*/ - -All the code inside the Python block is executed at the -time it's parsed. All text written to stdout inside the block -is redirected into the "output" after the block. - -As an example, here's a Python block that adds a static integer -variable to the C code:: - - /*[python input] - print('static int __ignored_unused_variable__ = 0;') - [python start generated code]*/ - static int __ignored_unused_variable__ = 0; - /*[python checksum:...]*/ - - -Using a "self converter" ------------------------- - -Argument Clinic automatically adds a "self" parameter for you -using a default converter. It automatically sets the ``type`` -of this parameter to the "pointer to an instance" you specified -when you declared the type. However, you can override -Argument Clinic's converter and specify one yourself. -Just add your own ``self`` parameter as the first parameter in a -block, and ensure that its converter is an instance of -``self_converter`` or a subclass thereof. - -What's the point? This lets you override the type of ``self``, -or give it a different default name. - -How do you specify the custom type you want to cast ``self`` to? -If you only have one or two functions with the same type for ``self``, -you can directly use Argument Clinic's existing ``self`` converter, -passing in the type you want to use as the ``type`` parameter:: - - /*[clinic input] - - _pickle.Pickler.dump - - self: self(type="PicklerObject *") - obj: object - / - - Write a pickled representation of the given object to the open file. - [clinic start generated code]*/ - -On the other hand, if you have a lot of functions that will use the same -type for ``self``, it's best to create your own converter, subclassing -``self_converter`` but overwriting the ``type`` member:: - - /*[python input] - class PicklerObject_converter(self_converter): - type = "PicklerObject *" - [python start generated code]*/ - - /*[clinic input] - - _pickle.Pickler.dump - - self: PicklerObject - obj: object - / - - Write a pickled representation of the given object to the open file. - [clinic start generated code]*/ - - -Using a "defining class" converter ----------------------------------- - -Argument Clinic facilitates gaining access to the defining class of a method. -This is useful for :ref:`heap type ` methods that need to fetch -module level state. Use :c:func:`PyType_FromModuleAndSpec` to associate a new -heap type with a module. You can now use :c:func:`PyType_GetModuleState` on -the defining class to fetch the module state, for example from a module method. - -Example from ``Modules/zlibmodule.c``. First, ``defining_class`` is added to -the clinic input:: - - /*[clinic input] - zlib.Compress.compress - - cls: defining_class - data: Py_buffer - Binary data to be compressed. - / - - -After running the Argument Clinic tool, the following function signature is -generated:: - - /*[clinic start generated code]*/ - static PyObject * - zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, - Py_buffer *data) - /*[clinic end generated code: output=6731b3f0ff357ca6 input=04d00f65ab01d260]*/ - - -The following code can now use ``PyType_GetModuleState(cls)`` to fetch the -module state:: - - zlibstate *state = PyType_GetModuleState(cls); - - -Each method may only have one argument using this converter, and it must appear -after ``self``, or, if ``self`` is not used, as the first argument. The argument -will be of type ``PyTypeObject *``. The argument will not appear in the -``__text_signature__``. - -The ``defining_class`` converter is not compatible with ``__init__`` and ``__new__`` -methods, which cannot use the ``METH_METHOD`` convention. - -It is not possible to use ``defining_class`` with slot methods. In order to -fetch the module state from such methods, use :c:func:`PyType_GetModuleByDef` -to look up the module and then :c:func:`PyModule_GetState` to fetch the module -state. Example from the ``setattro`` slot method in -``Modules/_threadmodule.c``:: - - static int - local_setattro(localobject *self, PyObject *name, PyObject *v) - { - PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &thread_module); - thread_module_state *state = get_thread_state(module); - ... - } - - -See also :pep:`573`. - - -Writing a custom converter --------------------------- - -As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from ``CConverter``. -The main purpose of a custom converter is if you have a parameter using -the ``O&`` format unit—parsing this parameter means calling -a :c:func:`PyArg_ParseTuple` "converter function". - -Your converter class should be named ``*something*_converter``. -If the name follows this convention, then your converter class -will be automatically registered with Argument Clinic; its name -will be the name of your class with the ``_converter`` suffix -stripped off. (This is accomplished with a metaclass.) - -You shouldn't subclass ``CConverter.__init__``. Instead, you should -write a ``converter_init()`` function. ``converter_init()`` -always accepts a ``self`` parameter; after that, all additional -parameters *must* be keyword-only. Any arguments passed in to -the converter in Argument Clinic will be passed along to your -``converter_init()``. - -There are some additional members of ``CConverter`` you may wish -to specify in your subclass. Here's the current list: - -``type`` - The C type to use for this variable. - ``type`` should be a Python string specifying the type, e.g. ``int``. - If this is a pointer type, the type string should end with ``' *'``. - -``default`` - The Python default value for this parameter, as a Python value. - Or the magic value ``unspecified`` if there is no default. - -``py_default`` - ``default`` as it should appear in Python code, - as a string. - Or ``None`` if there is no default. - -``c_default`` - ``default`` as it should appear in C code, - as a string. - Or ``None`` if there is no default. - -``c_ignored_default`` - The default value used to initialize the C variable when - there is no default, but not specifying a default may - result in an "uninitialized variable" warning. This can - easily happen when using option groups—although - properly written code will never actually use this value, - the variable does get passed in to the impl, and the - C compiler will complain about the "use" of the - uninitialized value. This value should always be a - non-empty string. - -``converter`` - The name of the C converter function, as a string. - -``impl_by_reference`` - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into the impl function. - -``parse_by_reference`` - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into :c:func:`PyArg_ParseTuple`. - - -Here's the simplest example of a custom converter, from ``Modules/zlibmodule.c``:: - - /*[python input] - - class ssize_t_converter(CConverter): - type = 'Py_ssize_t' - converter = 'ssize_t_converter' - - [python start generated code]*/ - /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ - -This block adds a converter to Argument Clinic named ``ssize_t``. Parameters -declared as ``ssize_t`` will be declared as type :c:type:`Py_ssize_t`, and will -be parsed by the ``'O&'`` format unit, which will call the -``ssize_t_converter`` converter function. ``ssize_t`` variables -automatically support default values. - -More sophisticated custom converters can insert custom C code to -handle initialization and cleanup. -You can see more examples of custom converters in the CPython -source tree; grep the C files for the string ``CConverter``. - -Writing a custom return converter ---------------------------------- - -Writing a custom return converter is much like writing -a custom converter. Except it's somewhat simpler, because return -converters are themselves much simpler. - -Return converters must subclass ``CReturnConverter``. -There are no examples yet of custom return converters, -because they are not widely used yet. If you wish to -write your own return converter, please read ``Tools/clinic/clinic.py``, -specifically the implementation of ``CReturnConverter`` and -all its subclasses. - -METH_O and METH_NOARGS ----------------------------------------------- - -To convert a function using ``METH_O``, make sure the function's -single argument is using the ``object`` converter, and mark the -arguments as positional-only:: - - /*[clinic input] - meth_o_sample - - argument: object - / - [clinic start generated code]*/ - - -To convert a function using ``METH_NOARGS``, just don't specify -any arguments. - -You can still use a self converter, a return converter, and specify -a ``type`` argument to the object converter for ``METH_O``. - -tp_new and tp_init functions ----------------------------------------------- - -You can convert ``tp_new`` and ``tp_init`` functions. Just name -them ``__new__`` or ``__init__`` as appropriate. Notes: - -* The function name generated for ``__new__`` doesn't end in ``__new__`` - like it would by default. It's just the name of the class, converted - into a valid C identifier. - -* No ``PyMethodDef`` ``#define`` is generated for these functions. - -* ``__init__`` functions return ``int``, not ``PyObject *``. - -* Use the docstring as the class docstring. - -* Although ``__new__`` and ``__init__`` functions must always - accept both the ``args`` and ``kwargs`` objects, when converting - you may specify any signature for these functions that you like. - (If your function doesn't support keywords, the parsing function - generated will throw an exception if it receives any.) - -Changing and redirecting Clinic's output ----------------------------------------- - -It can be inconvenient to have Clinic's output interspersed with -your conventional hand-edited C code. Luckily, Clinic is configurable: -you can buffer up its output for printing later (or earlier!), or write -its output to a separate file. You can also add a prefix or suffix to -every line of Clinic's generated output. - -While changing Clinic's output in this manner can be a boon to readability, -it may result in Clinic code using types before they are defined, or -your code attempting to use Clinic-generated code before it is defined. -These problems can be easily solved by rearranging the declarations in your file, -or moving where Clinic's generated code goes. (This is why the default behavior -of Clinic is to output everything into the current block; while many people -consider this hampers readability, it will never require rearranging your -code to fix definition-before-use problems.) - -Let's start with defining some terminology: - -*field* - A field, in this context, is a subsection of Clinic's output. - For example, the ``#define`` for the ``PyMethodDef`` structure - is a field, called ``methoddef_define``. Clinic has seven - different fields it can output per function definition: - - .. code-block:: none - - docstring_prototype - docstring_definition - methoddef_define - impl_prototype - parser_prototype - parser_definition - impl_definition - - All the names are of the form ``"_"``, - where ``""`` is the semantic object represented (the parsing function, - the impl function, the docstring, or the methoddef structure) and ``""`` - represents what kind of statement the field is. Field names that end in - ``"_prototype"`` - represent forward declarations of that thing, without the actual body/data - of the thing; field names that end in ``"_definition"`` represent the actual - definition of the thing, with the body/data of the thing. (``"methoddef"`` - is special, it's the only one that ends with ``"_define"``, representing that - it's a preprocessor #define.) - -*destination* - A destination is a place Clinic can write output to. There are - five built-in destinations: - - ``block`` - The default destination: printed in the output section of - the current Clinic block. - - ``buffer`` - A text buffer where you can save text for later. Text sent - here is appended to the end of any existing text. It's an - error to have any text left in the buffer when Clinic finishes - processing a file. - - ``file`` - A separate "clinic file" that will be created automatically by Clinic. - The filename chosen for the file is ``{basename}.clinic{extension}``, - where ``basename`` and ``extension`` were assigned the output - from ``os.path.splitext()`` run on the current file. (Example: - the ``file`` destination for ``_pickle.c`` would be written to - ``_pickle.clinic.c``.) - - **Important: When using a** ``file`` **destination, you** - *must check in* **the generated file!** - - ``two-pass`` - A buffer like ``buffer``. However, a two-pass buffer can only - be dumped once, and it prints out all text sent to it during - all processing, even from Clinic blocks *after* the dumping point. - - ``suppress`` - The text is suppressed—thrown away. - - -Clinic defines five new directives that let you reconfigure its output. - -The first new directive is ``dump``: - -.. code-block:: none - - dump - -This dumps the current contents of the named destination into the output of -the current block, and empties it. This only works with ``buffer`` and -``two-pass`` destinations. - -The second new directive is ``output``. The most basic form of ``output`` -is like this: - -.. code-block:: none - - output - -This tells Clinic to output *field* to *destination*. ``output`` also -supports a special meta-destination, called ``everything``, which tells -Clinic to output *all* fields to that *destination*. - -``output`` has a number of other functions: - -.. code-block:: none - - output push - output pop - output preset - - -``output push`` and ``output pop`` allow you to push and pop -configurations on an internal configuration stack, so that you -can temporarily modify the output configuration, then easily restore -the previous configuration. Simply push before your change to save -the current configuration, then pop when you wish to restore the -previous configuration. - -``output preset`` sets Clinic's output to one of several built-in -preset configurations, as follows: - - ``block`` - Clinic's original starting configuration. Writes everything - immediately after the input block. - - Suppress the ``parser_prototype`` - and ``docstring_prototype``, write everything else to ``block``. - - ``file`` - Designed to write everything to the "clinic file" that it can. - You then ``#include`` this file near the top of your file. - You may need to rearrange your file to make this work, though - usually this just means creating forward declarations for various - ``typedef`` and ``PyTypeObject`` definitions. - - Suppress the ``parser_prototype`` - and ``docstring_prototype``, write the ``impl_definition`` to - ``block``, and write everything else to ``file``. - - The default filename is ``"{dirname}/clinic/{basename}.h"``. - - ``buffer`` - Save up most of the output from Clinic, to be written into - your file near the end. For Python files implementing modules - or builtin types, it's recommended that you dump the buffer - just above the static structures for your module or - builtin type; these are normally very near the end. Using - ``buffer`` may require even more editing than ``file``, if - your file has static ``PyMethodDef`` arrays defined in the - middle of the file. - - Suppress the ``parser_prototype``, ``impl_prototype``, - and ``docstring_prototype``, write the ``impl_definition`` to - ``block``, and write everything else to ``file``. - - ``two-pass`` - Similar to the ``buffer`` preset, but writes forward declarations to - the ``two-pass`` buffer, and definitions to the ``buffer``. - This is similar to the ``buffer`` preset, but may require - less editing than ``buffer``. Dump the ``two-pass`` buffer - near the top of your file, and dump the ``buffer`` near - the end just like you would when using the ``buffer`` preset. - - Suppresses the ``impl_prototype``, write the ``impl_definition`` - to ``block``, write ``docstring_prototype``, ``methoddef_define``, - and ``parser_prototype`` to ``two-pass``, write everything else - to ``buffer``. - - ``partial-buffer`` - Similar to the ``buffer`` preset, but writes more things to ``block``, - only writing the really big chunks of generated code to ``buffer``. - This avoids the definition-before-use problem of ``buffer`` completely, - at the small cost of having slightly more stuff in the block's output. - Dump the ``buffer`` near the end, just like you would when using - the ``buffer`` preset. - - Suppresses the ``impl_prototype``, write the ``docstring_definition`` - and ``parser_definition`` to ``buffer``, write everything else to ``block``. - -The third new directive is ``destination``: - -.. code-block:: none - - destination [...] - -This performs an operation on the destination named ``name``. - -There are two defined subcommands: ``new`` and ``clear``. - -The ``new`` subcommand works like this: - -.. code-block:: none - - destination new - -This creates a new destination with name ```` and type ````. - -There are five destination types: - - ``suppress`` - Throws the text away. - - ``block`` - Writes the text to the current block. This is what Clinic - originally did. - - ``buffer`` - A simple text buffer, like the "buffer" builtin destination above. - - ``file`` - A text file. The file destination takes an extra argument, - a template to use for building the filename, like so: - - destination new - - The template can use three strings internally that will be replaced - by bits of the filename: - - {path} - The full path to the file, including directory and full filename. - {dirname} - The name of the directory the file is in. - {basename} - Just the name of the file, not including the directory. - {basename_root} - Basename with the extension clipped off - (everything up to but not including the last '.'). - {basename_extension} - The last '.' and everything after it. If the basename - does not contain a period, this will be the empty string. - - If there are no periods in the filename, {basename} and {filename} - are the same, and {extension} is empty. "{basename}{extension}" - is always exactly the same as "{filename}"." - - ``two-pass`` - A two-pass buffer, like the "two-pass" builtin destination above. - - -The ``clear`` subcommand works like this: - -.. code-block:: none - - destination clear - -It removes all the accumulated text up to this point in the destination. -(I don't know what you'd need this for, but I thought maybe it'd be -useful while someone's experimenting.) - -The fourth new directive is ``set``: - -.. code-block:: none - - set line_prefix "string" - set line_suffix "string" - -``set`` lets you set two internal variables in Clinic. -``line_prefix`` is a string that will be prepended to every line of Clinic's output; -``line_suffix`` is a string that will be appended to every line of Clinic's output. - -Both of these support two format strings: - - ``{block comment start}`` - Turns into the string ``/*``, the start-comment text sequence for C files. - - ``{block comment end}`` - Turns into the string ``*/``, the end-comment text sequence for C files. - -The final new directive is one you shouldn't need to use directly, -called ``preserve``: - -.. code-block:: none - - preserve - -This tells Clinic that the current contents of the output should be kept, unmodified. -This is used internally by Clinic when dumping output into ``file`` files; wrapping -it in a Clinic block lets Clinic use its existing checksum functionality to ensure -the file was not modified by hand before it gets overwritten. - - -The #ifdef trick ----------------------------------------------- - -If you're converting a function that isn't available on all platforms, -there's a trick you can use to make life a little easier. The existing -code probably looks like this:: - - #ifdef HAVE_FUNCTIONNAME - static module_functionname(...) - { - ... - } - #endif /* HAVE_FUNCTIONNAME */ - -And then in the ``PyMethodDef`` structure at the bottom the existing code -will have: - -.. code-block:: none - - #ifdef HAVE_FUNCTIONNAME - {'functionname', ... }, - #endif /* HAVE_FUNCTIONNAME */ - -In this scenario, you should enclose the body of your impl function inside the ``#ifdef``, -like so:: - - #ifdef HAVE_FUNCTIONNAME - /*[clinic input] - module.functionname - ... - [clinic start generated code]*/ - static module_functionname(...) - { - ... - } - #endif /* HAVE_FUNCTIONNAME */ - -Then, remove those three lines from the ``PyMethodDef`` structure, -replacing them with the macro Argument Clinic generated: - -.. code-block:: none - - MODULE_FUNCTIONNAME_METHODDEF - -(You can find the real name for this macro inside the generated code. -Or you can calculate it yourself: it's the name of your function as defined -on the first line of your block, but with periods changed to underscores, -uppercased, and ``"_METHODDEF"`` added to the end.) - -Perhaps you're wondering: what if ``HAVE_FUNCTIONNAME`` isn't defined? -The ``MODULE_FUNCTIONNAME_METHODDEF`` macro won't be defined either! - -Here's where Argument Clinic gets very clever. It actually detects that the -Argument Clinic block might be deactivated by the ``#ifdef``. When that -happens, it generates a little extra code that looks like this:: - - #ifndef MODULE_FUNCTIONNAME_METHODDEF - #define MODULE_FUNCTIONNAME_METHODDEF - #endif /* !defined(MODULE_FUNCTIONNAME_METHODDEF) */ - -That means the macro always works. If the function is defined, this turns -into the correct structure, including the trailing comma. If the function is -undefined, this turns into nothing. - -However, this causes one ticklish problem: where should Argument Clinic put this -extra code when using the "block" output preset? It can't go in the output block, -because that could be deactivated by the ``#ifdef``. (That's the whole point!) - -In this situation, Argument Clinic writes the extra code to the "buffer" destination. -This may mean that you get a complaint from Argument Clinic: - -.. code-block:: none - - Warning in file "Modules/posixmodule.c" on line 12357: - Destination buffer 'buffer' not empty at end of file, emptying. - -When this happens, just open your file, find the ``dump buffer`` block that -Argument Clinic added to your file (it'll be at the very bottom), then -move it above the ``PyMethodDef`` structure where that macro is used. - - - -Using Argument Clinic in Python files -------------------------------------- - -It's actually possible to use Argument Clinic to preprocess Python files. -There's no point to using Argument Clinic blocks, of course, as the output -wouldn't make any sense to the Python interpreter. But using Argument Clinic -to run Python blocks lets you use Python as a Python preprocessor! - -Since Python comments are different from C comments, Argument Clinic -blocks embedded in Python files look slightly different. They look like this: - -.. code-block:: python3 - - #/*[python input] - #print("def foo(): pass") - #[python start generated code]*/ - def foo(): pass - #/*[python checksum:...]*/ + The Argument Clinic How-TO has been moved to the `Python Developer's Guide + `__. diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index a3068d86d85bc4..4828e2fa29bd24 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -527,7 +527,7 @@ If you're in doubt about the detailed behavior of the curses functions, consult the manual pages for your curses implementation, whether it's ncurses or a proprietary Unix vendor's. The manual pages will document any quirks, and provide complete lists of all the -functions, attributes, and :const:`ACS_\*` characters available to +functions, attributes, and :ref:`ACS_\* ` characters available to you. Because the curses API is so large, some functions aren't supported in diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 3688c47f0d6ec9..f732aaea729d40 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -521,11 +521,11 @@ everyday Python programs. Descriptor protocol ------------------- -``descr.__get__(self, obj, type=None) -> value`` +``descr.__get__(self, obj, type=None)`` -``descr.__set__(self, obj, value) -> None`` +``descr.__set__(self, obj, value)`` -``descr.__delete__(self, obj) -> None`` +``descr.__delete__(self, obj)`` That is all there is to it. Define any of these methods and an object is considered a descriptor and can override default behavior upon being looked up @@ -779,8 +779,8 @@ by a search through the class's :term:`method resolution order`. If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. -The full C implementation can be found in :c:func:`type_getattro()` and -:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. +The full C implementation can be found in :c:func:`!type_getattro` and +:c:func:`!_PyType_Lookup` in :source:`Objects/typeobject.c`. Invocation from super @@ -794,7 +794,7 @@ for the base class ``B`` immediately following ``A`` and then returns ``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned unchanged. -The full C implementation can be found in :c:func:`super_getattro()` in +The full C implementation can be found in :c:func:`!super_getattro` in :source:`Objects/typeobject.c`. A pure Python equivalent can be found in `Guido's Tutorial `_. @@ -836,8 +836,8 @@ and if they define :meth:`__set_name__`, that method is called with two arguments. The *owner* is the class where the descriptor is used, and the *name* is the class variable the descriptor was assigned to. -The implementation details are in :c:func:`type_new()` and -:c:func:`set_names()` in :source:`Objects/typeobject.c`. +The implementation details are in :c:func:`!type_new` and +:c:func:`!set_names` in :source:`Objects/typeobject.c`. Since the update logic is in :meth:`type.__new__`, notifications only take place at the time of class creation. If descriptors are added to the class @@ -943,6 +943,10 @@ it can be updated: >>> Movie('Star Wars').director 'J.J. Abrams' +.. testcleanup:: + + conn.close() + Pure Python Equivalents ^^^^^^^^^^^^^^^^^^^^^^^ @@ -1009,17 +1013,23 @@ here is a pure Python equivalent: if obj is None: return self if self.fget is None: - raise AttributeError(f"property '{self._name}' has no getter") + raise AttributeError( + f'property {self._name!r} of {type(obj).__name__!r} object has no getter' + ) return self.fget(obj) def __set__(self, obj, value): if self.fset is None: - raise AttributeError(f"property '{self._name}' has no setter") + raise AttributeError( + f'property {self._name!r} of {type(obj).__name__!r} object has no setter' + ) self.fset(obj, value) def __delete__(self, obj): if self.fdel is None: - raise AttributeError(f"property '{self._name}' has no deleter") + raise AttributeError( + f'property {self._name!r} of {type(obj).__name__!r} object has no deleter' + ) self.fdel(obj) def getter(self, fget): @@ -1050,6 +1060,11 @@ here is a pure Python equivalent: def delx(self): del self.__x x = Property(getx, setx, delx, "I'm the 'x' property.") + no_getter = Property(None, setx, delx, "I'm the 'x' property.") + no_setter = Property(getx, None, delx, "I'm the 'x' property.") + no_deleter = Property(getx, setx, None, "I'm the 'x' property.") + no_doc = Property(getx, setx, delx, None) + # Now do it again but use the decorator style @@ -1088,6 +1103,32 @@ here is a pure Python equivalent: >>> hasattr(ccc, 'x') False + >>> cc = CC() + >>> cc.x = 33 + >>> try: + ... cc.no_getter + ... except AttributeError as e: + ... e.args[0] + ... + "property 'no_getter' of 'CC' object has no getter" + + >>> try: + ... cc.no_setter = 33 + ... except AttributeError as e: + ... e.args[0] + ... + "property 'no_setter' of 'CC' object has no setter" + + >>> try: + ... del cc.no_deleter + ... except AttributeError as e: + ... e.args[0] + ... + "property 'no_deleter' of 'CC' object has no deleter" + + >>> CC.no_doc.__doc__ is None + True + The :func:`property` builtin helps whenever a user interface has granted attribute access and then subsequent changes require the intervention of a method. @@ -1141,6 +1182,16 @@ roughly equivalent to: obj = self.__self__ return func(obj, *args, **kwargs) + def __getattribute__(self, name): + "Emulate method_getset() in Objects/classobject.c" + if name == '__doc__': + return self.__func__.__doc__ + return object.__getattribute__(self, name) + + def __getattr__(self, name): + "Emulate method_getattro() in Objects/classobject.c" + return getattr(self.__func__, name) + To support automatic creation of methods, functions include the :meth:`__get__` method for binding methods during attribute access. This means that functions are non-data descriptors that return bound methods @@ -1420,10 +1471,6 @@ Using the non-data descriptor protocol, a pure Python version of def __get__(self, obj, cls=None): if cls is None: cls = type(obj) - if hasattr(type(self.f), '__get__'): - # This code path was added in Python 3.9 - # and was deprecated in Python 3.11. - return self.f.__get__(cls, cls) return MethodType(self.f, cls) .. testcode:: @@ -1436,11 +1483,6 @@ Using the non-data descriptor protocol, a pure Python version of "Class method that returns a tuple" return (cls.__name__, x, y) - @ClassMethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - .. doctest:: :hide: @@ -1453,10 +1495,6 @@ Using the non-data descriptor protocol, a pure Python version of >>> t.cm(11, 22) ('T', 11, 22) - # Check the alternate path for chained descriptors - >>> T.__doc__ - "A doc for 'T'" - # Verify that T uses our emulation >>> type(vars(T)['cm']).__name__ 'ClassMethod' @@ -1481,24 +1519,6 @@ Using the non-data descriptor protocol, a pure Python version of ('T', 11, 22) -The code path for ``hasattr(type(self.f), '__get__')`` was added in -Python 3.9 and makes it possible for :func:`classmethod` to support -chained decorators. For example, a classmethod and property could be -chained together. In Python 3.11, this functionality was deprecated. - -.. testcode:: - - class G: - @classmethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - -.. doctest:: - - >>> G.__doc__ - "A doc for 'G'" - The :func:`functools.update_wrapper` call in ``ClassMethod`` adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 68b75c529e92c7..1e9ac9b6761b64 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -426,10 +426,17 @@ enumeration, with the exception of special methods (:meth:`__str__`, :meth:`__add__`, etc.), descriptors (methods are also descriptors), and variable names listed in :attr:`_ignore_`. -Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then +Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__`, any value(s) given to the enum member will be passed into those methods. See `Planet`_ for an example. +.. note:: + + The :meth:`__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`__new__` which is used after + class creation for lookup of existing members. See :ref:`new-vs-init` for + more details. + Restricted Enum subclassing --------------------------- @@ -476,6 +483,7 @@ Dataclass support When inheriting from a :class:`~dataclasses.dataclass`, the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: + >>> from dataclasses import dataclass, field >>> @dataclass ... class CreatureDataMixin: ... size: str @@ -517,7 +525,17 @@ from that module. nested in other classes. It is possible to modify how enum members are pickled/unpickled by defining -:meth:`__reduce_ex__` in the enumeration class. +:meth:`__reduce_ex__` in the enumeration class. The default method is by-value, +but enums with complicated values may want to use by-name:: + + >>> import enum + >>> class MyEnum(enum.Enum): + ... __reduce_ex__ = enum.pickle_by_enum_name + +.. note:: + + Using by-name for flags is not recommended, as unnamed aliases will + not unpickle. Functional API @@ -589,9 +607,9 @@ The complete signature is:: start=1, ) -:value: What the new enum class will record as its name. +* *value*: What the new enum class will record as its name. -:names: The enum members. This can be a whitespace- or comma-separated string +* *names*: The enum members. This can be a whitespace- or comma-separated string (values will start at 1 unless otherwise specified):: 'RED GREEN BLUE' | 'RED,GREEN,BLUE' | 'RED, GREEN, BLUE' @@ -608,13 +626,13 @@ The complete signature is:: {'CHARTREUSE': 7, 'SEA_GREEN': 11, 'ROSEMARY': 42} -:module: name of module where new enum class can be found. +* *module*: name of module where new enum class can be found. -:qualname: where in module new enum class can be found. +* *qualname*: where in module new enum class can be found. -:type: type to mix in to new enum class. +* *type*: type to mix in to new enum class. -:start: number to start counting at if only names are passed in. +* *start*: number to start counting at if only names are passed in. .. versionchanged:: 3.5 The *start* parameter was added. @@ -754,7 +772,7 @@ be combined with them (but may lose :class:`IntFlag` membership:: >>> Perm.X | 4 - >>> Perm.X | 8 + >>> Perm.X + 8 9 .. note:: @@ -850,7 +868,7 @@ Others While :class:`IntEnum` is part of the :mod:`enum` module, it would be very simple to implement independently:: - class IntEnum(int, Enum): + class IntEnum(int, ReprEnum): # or Enum instead of ReprEnum pass This demonstrates how similar derived enumerations can be defined; for example @@ -858,8 +876,8 @@ a :class:`FloatEnum` that mixes in :class:`float` instead of :class:`int`. Some rules: -1. When subclassing :class:`Enum`, mix-in types must appear before - :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum` +1. When subclassing :class:`Enum`, mix-in types must appear before the + :class:`Enum` class itself in the sequence of bases, as in the :class:`IntEnum` example above. 2. Mix-in types must be subclassable. For example, :class:`bool` and :class:`range` are not subclassable and will throw an error during Enum @@ -886,6 +904,8 @@ Some rules: :meth:`__str__` method has been reset to their data types' :meth:`__str__` method. +.. _new-vs-init: + When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------ @@ -918,6 +938,11 @@ want one of them to be the value:: >>> print(Coordinate(3)) Coordinate.VY +.. warning:: + + *Do not* call ``super().__new__()``, as the lookup-only ``__new__`` is the one + that is found; instead, use the data type directly. + Finer Points ^^^^^^^^^^^^ @@ -936,30 +961,34 @@ all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` +- :attr:`~Enum._name_` -- name of the member +- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` +- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; + may be overridden +- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a + :class:`str`, that will not be transformed into members, and will be removed + from the final class +- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for + an enum member; may be overridden +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an + existing member. See `MultiValueEnum`_ for an example. -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, - that will not be transformed into members, and will be removed from the final - class -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used by the `Functional API`_ and by - :class:`auto` to get an appropriate value for an enum member; may be - overridden + .. note:: -.. note:: + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. - For standard :class:`Enum` classes the next value chosen is the last value seen - incremented by one. + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two. - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two, regardless of the last value seen. + .. versionchanged:: 3.13 + Prior versions would use the last seen value instead of the highest value. .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can be provided. It will be checked against the actual order of the enumeration @@ -1133,13 +1162,14 @@ the following are true: There is a new boundary mechanism that controls how out-of-range / invalid bits are handled: ``STRICT``, ``CONFORM``, ``EJECT``, and ``KEEP``: - * STRICT --> raises an exception when presented with invalid values - * CONFORM --> discards any invalid bits - * EJECT --> lose Flag status and become a normal int with the given value - * KEEP --> keep the extra bits - - keeps Flag status and extra bits - - extra bits do not show up in iteration - - extra bits do show up in repr() and str() +* STRICT --> raises an exception when presented with invalid values +* CONFORM --> discards any invalid bits +* EJECT --> lose Flag status and become a normal int with the given value +* KEEP --> keep the extra bits + + - keeps Flag status and extra bits + - extra bits do not show up in iteration + - extra bits do show up in repr() and str() The default for Flag is ``STRICT``, the default for ``IntFlag`` is ``EJECT``, and the default for ``_convert_`` is ``KEEP`` (see ``ssl.Options`` for an @@ -1344,6 +1374,13 @@ to handle any extra arguments:: members; it is then replaced by Enum's :meth:`__new__` which is used after class creation for lookup of existing members. +.. warning:: + + *Do not* call ``super().__new__()``, as the lookup-only ``__new__`` is the one + that is found; instead, use the data type directly -- e.g.:: + + obj = int.__new__(cls, value) + OrderedEnum ^^^^^^^^^^^ @@ -1404,7 +1441,7 @@ alias:: ... GRENE = 2 ... Traceback (most recent call last): - ... + ... ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN' .. note:: @@ -1414,6 +1451,29 @@ alias:: disallowing aliases, the :func:`unique` decorator can be used instead. +MultiValueEnum +^^^^^^^^^^^^^^^^^ + +Supports having more than one value per member:: + + >>> class MultiValueEnum(Enum): + ... def __new__(cls, value, *values): + ... self = object.__new__(cls) + ... self._value_ = value + ... for v in values: + ... self._add_value_alias_(v) + ... return self + ... + >>> class DType(MultiValueEnum): + ... float32 = 'f', 8 + ... double64 = 'd', 9 + ... + >>> DType('f') + + >>> DType(9) + + + Planet ^^^^^^ diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 5cf12cc52bde4e..b0f9d22d74f0e3 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1072,8 +1072,8 @@ write the obvious :keyword:`for` loop:: A related function is :func:`itertools.accumulate(iterable, func=operator.add) `. It performs the same calculation, but instead of -returning only the final result, :func:`accumulate` returns an iterator that -also yields each partial result:: +returning only the final result, :func:`~itertools.accumulate` returns an iterator +that also yields each partial result:: itertools.accumulate([1, 2, 3, 4, 5]) => 1, 3, 6, 10, 15 diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index f521276a5a83c5..a835bb5f13bd1c 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -28,9 +28,9 @@ Currently, the HOWTOs are: urllib2.rst argparse.rst ipaddress.rst - clinic.rst instrumentation.rst perf_profiling.rst annotations.rst isolating-extensions.rst + timerfd.rst diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 4ce15c69dac90b..9c99fcecce1fcb 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -13,9 +13,9 @@ DTrace and SystemTap are monitoring tools, each providing a way to inspect what the processes on a computer system are doing. They both use domain-specific languages allowing a user to write scripts which: - - filter which processes are to be observed - - gather data from the processes of interest - - generate reports on the data +- filter which processes are to be observed +- gather data from the processes of interest +- generate reports on the data As of Python 3.6, CPython can be built with embedded "markers", also known as "probes", that can be observed by a DTrace or SystemTap script, @@ -246,11 +246,9 @@ The output looks like this: where the columns are: - - time in microseconds since start of script - - - name of executable - - - PID of process +- time in microseconds since start of script +- name of executable +- PID of process and the remainder indicates the call/return hierarchy as the script executes. @@ -292,11 +290,11 @@ Available static markers .. object:: function__return(str filename, str funcname, int lineno) - This marker is the converse of :c:func:`function__entry`, and indicates that + This marker is the converse of :c:func:`!function__entry`, and indicates that execution of a Python function has ended (either via ``return``, or via an exception). It is only triggered for pure-Python (bytecode) functions. - The arguments are the same as for :c:func:`function__entry` + The arguments are the same as for :c:func:`!function__entry` .. object:: line(str filename, str funcname, int lineno) @@ -304,7 +302,7 @@ Available static markers the equivalent of line-by-line tracing with a Python profiler. It is not triggered within C functions. - The arguments are the same as for :c:func:`function__entry`. + The arguments are the same as for :c:func:`!function__entry`. .. object:: gc__start(int generation) diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 0262054ae2b4a0..835c0afad7c6c8 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -1,5 +1,7 @@ .. highlight:: c +.. _isolating-extensions-howto: + *************************** Isolating Extension Modules *************************** @@ -62,7 +64,7 @@ Enter Per-Module State Instead of focusing on per-interpreter state, Python's C API is evolving to better support the more granular *per-module* state. -This means that C-level data is be attached to a *module object*. +This means that C-level data should be attached to a *module object*. Each interpreter creates its own module object, keeping the data separate. For testing the isolation, multiple module objects corresponding to a single extension can even be loaded in a single interpreter. @@ -298,10 +300,10 @@ Watch out for the following two points in particular (but note that this is not a comprehensive list): * Unlike static types, heap type objects are mutable by default. - Use the :c:data:`Py_TPFLAGS_IMMUTABLETYPE` flag to prevent mutability. + Use the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag to prevent mutability. * Heap types inherit :c:member:`~PyTypeObject.tp_new` by default, so it may become possible to instantiate them from Python code. - You can prevent this with the :c:data:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag. + You can prevent this with the :c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag. Defining Heap Types @@ -333,16 +335,48 @@ To avoid memory leaks, instances of heap types must implement the garbage collection protocol. That is, heap types should: -- Have the :c:data:`Py_TPFLAGS_HAVE_GC` flag. +- Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - Define a traverse function using ``Py_tp_traverse``, which visits the type (e.g. using :c:expr:`Py_VISIT(Py_TYPE(self))`). -Please refer to the :ref:`the documentation ` of -:c:data:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse` +Please refer to the the documentation of +:c:macro:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse` for additional considerations. -If your traverse function delegates to the ``tp_traverse`` of its base class -(or another type), ensure that ``Py_TYPE(self)`` is visited only once. +The API for defining heap types grew organically, leaving it +somewhat awkward to use in its current state. +The following sections will guide you through common issues. + + +``tp_traverse`` in Python 3.8 and lower +....................................... + +The requirement to visit the type from ``tp_traverse`` was added in Python 3.9. +If you support Python 3.8 and lower, the traverse function must *not* +visit the type, so it must be more complicated:: + + static int my_traverse(PyObject *self, visitproc visit, void *arg) + { + if (Py_Version >= 0x03090000) { + Py_VISIT(Py_TYPE(self)); + } + return 0; + } + +Unfortunately, :c:data:`Py_Version` was only added in Python 3.11. +As a replacement, use: + +* :c:macro:`PY_VERSION_HEX`, if not using the stable ABI, or +* :py:data:`sys.version_info` (via :c:func:`PySys_GetObject` and + :c:func:`PyArg_ParseTuple`). + + +Delegating ``tp_traverse`` +.......................... + +If your traverse function delegates to the :c:member:`~PyTypeObject.tp_traverse` +of its base class (or another type), ensure that ``Py_TYPE(self)`` is visited +only once. Note that only heap type are expected to visit the type in ``tp_traverse``. For example, if your traverse function includes:: @@ -354,11 +388,70 @@ For example, if your traverse function includes:: if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) { // a heap type's tp_traverse already visited Py_TYPE(self) } else { - Py_VISIT(Py_TYPE(self)); + if (Py_Version >= 0x03090000) { + Py_VISIT(Py_TYPE(self)); + } } -It is not necessary to handle the type's reference count in ``tp_new`` -and ``tp_clear``. +It is not necessary to handle the type's reference count in +:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_clear`. + + +Defining ``tp_dealloc`` +....................... + +If your type has a custom :c:member:`~PyTypeObject.tp_dealloc` function, +it needs to: + +- call :c:func:`PyObject_GC_UnTrack` before any fields are invalidated, and +- decrement the reference count of the type. + +To keep the type valid while ``tp_free`` is called, the type's refcount needs +to be decremented *after* the instance is deallocated. For example:: + + static void my_dealloc(PyObject *self) + { + PyObject_GC_UnTrack(self); + ... + PyTypeObject *type = Py_TYPE(self); + type->tp_free(self); + Py_DECREF(type); + } + +The default ``tp_dealloc`` function does this, so +if your type does *not* override +``tp_dealloc`` you don't need to add it. + + +Not overriding ``tp_free`` +.......................... + +The :c:member:`~PyTypeObject.tp_free` slot of a heap type must be set to +:c:func:`PyObject_GC_Del`. +This is the default; do not override it. + + +Avoiding ``PyObject_New`` +......................... + +GC-tracked objects need to be allocated using GC-aware functions. + +If you use use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: + +- Get and call type's :c:member:`~PyTypeObject.tp_alloc` slot, if possible. + That is, replace ``TYPE *o = PyObject_New(TYPE, typeobj)`` with:: + + TYPE *o = typeobj->tp_alloc(typeobj, 0); + + Replace ``o = PyObject_NewVar(TYPE, typeobj, size)`` with the same, + but use size instead of the 0. + +- If the above is not possible (e.g. inside a custom ``tp_alloc``), + call :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`:: + + TYPE *o = PyObject_GC_New(TYPE, typeobj); + + TYPE *o = PyObject_GC_NewVar(TYPE, typeobj, size); Module State Access from Classes @@ -411,7 +504,7 @@ that subclass, which may be defined in different module than yours. pass For a method to get its "defining class", it must use the -:data:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS` +:ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS ` :c:type:`calling convention ` and the corresponding :c:type:`PyCMethod` signature:: @@ -461,13 +554,13 @@ Module State Access from Slot Methods, Getters and Setters .. After adding to limited API: - If you use the :ref:`limited API , + If you use the :ref:`limited API `, you must update ``Py_LIMITED_API`` to ``0x030b0000``, losing ABI compatibility with earlier versions. Slot methods—the fast C equivalents for special methods, such as :c:member:`~PyNumberMethods.nb_add` for :py:attr:`~object.__add__` or -:c:member:`~PyType.tp_new` for initialization—have a very simple API that +:c:member:`~PyTypeObject.tp_new` for initialization—have a very simple API that doesn't allow passing in the defining class, unlike with :c:type:`PyCMethod`. The same goes for getters and setters defined with :c:type:`PyGetSetDef`. @@ -483,14 +576,14 @@ to get the state:: return NULL; } -``PyType_GetModuleByDef`` works by searching the +:c:func:`!PyType_GetModuleByDef` works by searching the :term:`method resolution order` (i.e. all superclasses) for the first superclass that has a corresponding module. .. note:: In very exotic cases (inheritance chains spanning multiple modules - created from the same definition), ``PyType_GetModuleByDef`` might not + created from the same definition), :c:func:`!PyType_GetModuleByDef` might not return the module of the true defining class. However, it will always return a module with the same definition, ensuring a compatible C memory layout. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 6ef252d709e735..588f5a0a53ded0 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -761,7 +761,7 @@ printed on the console; on the server side, you should see something like: Note that there are some security issues with pickle in some scenarios. If these affect you, you can use an alternative serialization scheme by overriding -the :meth:`~handlers.SocketHandler.makePickle` method and implementing your +the :meth:`~SocketHandler.makePickle` method and implementing your alternative there, as well as adapting the above script to use your alternative serialization. @@ -835,6 +835,8 @@ To test these files, do the following in a POSIX environment: You may need to tweak the configuration files in the unlikely event that the configured ports clash with something else in your test environment. +.. currentmodule:: logging + .. _context-info: Adding contextual information to your logging output @@ -1546,7 +1548,7 @@ Sometimes you want to let a log file grow to a certain size, then open a new file and log to that. You may want to keep a certain number of these files, and when that many files have been created, rotate the files so that the number of files and the size of the files both remain bounded. For this usage pattern, the -logging package provides a :class:`~handlers.RotatingFileHandler`:: +logging package provides a :class:`RotatingFileHandler`:: import glob import logging @@ -1594,6 +1596,8 @@ and each time it reaches the size limit it is renamed with the suffix Obviously this example sets the log length much too small as an extreme example. You would want to set *maxBytes* to an appropriate value. +.. currentmodule:: logging + .. _format-styles: Use of alternative formatting styles @@ -1724,7 +1728,7 @@ when (and if) the logged message is actually about to be output to a log by a handler. So the only slightly unusual thing which might trip you up is that the parentheses go around the format string and the arguments, not just the format string. That's because the __ notation is just syntax sugar for a constructor -call to one of the XXXMessage classes. +call to one of the :samp:`{XXX}Message` classes. If you prefer, you can use a :class:`LoggerAdapter` to achieve a similar effect to the above, as in the following example:: @@ -1840,6 +1844,7 @@ However, it should be borne in mind that each link in the chain adds run-time overhead to all logging operations, and the technique should only be used when the use of a :class:`Filter` does not provide the desired result. +.. currentmodule:: logging.handlers .. _zeromq-handlers: @@ -1917,6 +1922,8 @@ of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: :ref:`A more advanced logging tutorial ` +.. currentmodule:: logging + An example dictionary-based configuration ----------------------------------------- @@ -2637,7 +2644,7 @@ when (and if) the logged message is actually about to be output to a log by a handler. So the only slightly unusual thing which might trip you up is that the parentheses go around the format string and the arguments, not just the format string. That’s because the __ notation is just syntax sugar for a constructor -call to one of the ``XXXMessage`` classes shown above. +call to one of the :samp:`{XXX}Message` classes shown above. .. _filters-dictconfig: @@ -3918,8 +3925,8 @@ that in other languages such as Java and C#, loggers are often static class attributes. However, this pattern doesn't make sense in Python, where the module (and not the class) is the unit of software decomposition. -Adding handlers other than :class:`NullHandler` to a logger in a library -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Adding handlers other than :class:`~logging.NullHandler` to a logger in a library +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Configuring logging by adding handlers, formatters and filters is the responsibility of the application developer, not the library developer. If you diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index a72e9a820ef347..7330cf675baa36 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -979,7 +979,7 @@ provided: #. :class:`NullHandler` instances do nothing with error messages. They are used by library developers who want to use logging, but want to avoid the 'No - handlers could be found for logger XXX' message which can be displayed if + handlers could be found for logger *XXX*' message which can be displayed if the library user has not configured logging. See :ref:`library-config` for more information. diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 6af5536166f58a..bb1c00e0aa51d5 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -24,7 +24,7 @@ functions to appear in the output of the ``perf`` profiler. When this mode is enabled, the interpreter will interpose a small piece of code compiled on the fly before the execution of every Python function and it will teach ``perf`` the relationship between this piece of code and the associated Python function using -`perf map files`_. +:doc:`perf map files <../c-api/perfmaps>`. .. note:: @@ -97,7 +97,7 @@ Then we can use ``perf report`` to analyze the data: | | | | | |--2.97%--_PyObject_Malloc ... -As you can see, the Python functions are not shown in the output, only ``_Py_Eval_EvalFrameDefault`` +As you can see, the Python functions are not shown in the output, only ``_PyEval_EvalFrameDefault`` (the function that evaluates the Python bytecode) shows up. Unfortunately that's not very useful because all Python functions use the same C function to evaluate bytecode so we cannot know which Python function corresponds to which bytecode-evaluating function. @@ -162,8 +162,7 @@ the :option:`!-X` option takes precedence over the environment variable. Example, using the environment variable:: - $ PYTHONPERFSUPPORT=1 - $ python script.py + $ PYTHONPERFSUPPORT=1 python script.py $ perf report -g -i perf.data Example, using the :option:`!-X` option:: @@ -206,5 +205,3 @@ You can check if your system has been compiled with this flag by running:: If you don't see any output it means that your interpreter has not been compiled with frame pointers and therefore it may not be able to show Python functions in the output of ``perf``. - -.. _perf map files: https://github.com/torvalds/linux/blob/0513e464f9007b70b96740271a948ca5ab6e7dd7/tools/perf/Documentation/jit-interface.txt diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index baea3e85c3b84b..501b16d82d4d6f 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -1,49 +1,48 @@ .. _pyporting-howto: -********************************* -Porting Python 2 Code to Python 3 -********************************* +************************************* +How to port Python 2 Code to Python 3 +************************************* :author: Brett Cannon .. topic:: Abstract - With Python 3 being the future of Python while Python 2 is still in active - use, it is good to have your project available for both major releases of - Python. This guide is meant to help you figure out how best to support both - Python 2 & 3 simultaneously. + Python 2 reached its official end-of-life at the start of 2020. This means + that no new bug reports, fixes, or changes will be made to Python 2 - it's + no longer supported. + + This guide is intended to provide you with a path to Python 3 for your + code, that includes compatibility with Python 2 as a first step. If you are looking to port an extension module instead of pure Python code, please see :ref:`cporting-howto`. - If you would like to read one core Python developer's take on why Python 3 - came into existence, you can read Nick Coghlan's `Python 3 Q & A`_ or - Brett Cannon's `Why Python 3 exists`_. - + The archived python-porting_ mailing list may contain some useful guidance. - For help with porting, you can view the archived python-porting_ mailing list. The Short Explanation ===================== -To make your project be single-source Python 2/3 compatible, the basic steps +To achieve Python 2/3 compatibility in a single code base, the basic steps are: #. Only worry about supporting Python 2.7 #. Make sure you have good test coverage (coverage.py_ can help; ``python -m pip install coverage``) -#. Learn the differences between Python 2 & 3 +#. Learn the differences between Python 2 and 3 #. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``) #. Use Pylint_ to help make sure you don't regress on your Python 3 support (``python -m pip install pylint``) #. Use caniusepython3_ to find out which of your dependencies are blocking your use of Python 3 (``python -m pip install caniusepython3``) #. Once your dependencies are no longer blocking you, use continuous integration - to make sure you stay compatible with Python 2 & 3 (tox_ can help test + to make sure you stay compatible with Python 2 and 3 (tox_ can help test against multiple versions of Python; ``python -m pip install tox``) -#. Consider using optional static type checking to make sure your type usage - works in both Python 2 & 3 (e.g. use mypy_ to check your typing under both - Python 2 & Python 3; ``python -m pip install mypy``). +#. Consider using optional :term:`static type checking ` + to make sure your type usage + works in both Python 2 and 3 (e.g. use mypy_ to check your typing under both + Python 2 and Python 3; ``python -m pip install mypy``). .. note:: @@ -55,43 +54,30 @@ are: Details ======= -A key point about supporting Python 2 & 3 simultaneously is that you can start -**today**! Even if your dependencies are not supporting Python 3 yet that does -not mean you can't modernize your code **now** to support Python 3. Most changes -required to support Python 3 lead to cleaner code using newer practices even in -Python 2 code. +Even if other factors - say, dependencies over which you have no control - +still require you to support Python 2, that does not prevent you taking the +step of including Python 3 support. -Another key point is that modernizing your Python 2 code to also support -Python 3 is largely automated for you. While you might have to make some API -decisions thanks to Python 3 clarifying text data versus binary data, the -lower-level work is now mostly done for you and thus can at least benefit from -the automated changes immediately. +Most changes required to support Python 3 lead to cleaner code using newer +practices even in Python 2 code. -Keep those key points in mind while you read on about the details of porting -your code to support Python 2 & 3 simultaneously. +Different versions of Python 2 +------------------------------ -Drop support for Python 2.6 and older -------------------------------------- +Ideally, your code should be compatible with Python 2.7, which was the +last supported version of Python 2. -While you can make Python 2.5 work with Python 3, it is **much** easier if you -only have to work with Python 2.7. If dropping Python 2.5 is not an -option then the six_ project can help you support Python 2.5 & 3 simultaneously -(``python -m pip install six``). Do realize, though, that nearly all the projects listed -in this HOWTO will not be available to you. +Some of the tools mentioned in this guide will not work with Python 2.6. -If you are able to skip Python 2.5 and older, then the required changes -to your code should continue to look and feel like idiomatic Python code. At -worst you will have to use a function instead of a method in some instances or -have to import a function instead of using a built-in one, but otherwise the -overall transformation should not feel foreign to you. +If absolutely necessary, the six_ project can help you support Python 2.5 and +3 simultaneously. Do realize, though, that nearly all the projects listed in +this guide will not be available to you. -But you should aim for only supporting Python 2.7. Python 2.6 is no longer -freely supported and thus is not receiving bugfixes. This means **you** will have -to work around any issues you come across with Python 2.6. There are also some -tools mentioned in this HOWTO which do not support Python 2.6 (e.g., Pylint_), -and this will become more commonplace as time goes on. It will simply be easier -for you if you only support the versions of Python that you have to support. +If you are able to skip Python 2.5 and older, the required changes to your +code will be minimal. At worst you will have to use a function instead of a +method in some instances or have to import a function instead of using a +built-in one. Make sure you specify the proper version support in your ``setup.py`` file @@ -118,62 +104,57 @@ coverage). If you don't already have a tool to measure test coverage then coverage.py_ is recommended. -Learn the differences between Python 2 & 3 -------------------------------------------- +Be aware of the differences between Python 2 and 3 +-------------------------------------------------- Once you have your code well-tested you are ready to begin porting your code to Python 3! But to fully understand how your code is going to change and what you want to look out for while you code, you will want to learn what changes -Python 3 makes in terms of Python 2. Typically the two best ways of doing that -is reading the :ref:`"What's New" ` doc for each release of Python 3 and the -`Porting to Python 3`_ book (which is free online). There is also a handy -`cheat sheet`_ from the Python-Future project. +Python 3 makes in terms of Python 2. + +Some resources for understanding the differences and their implications for you +code: + +* the :ref:`"What's New" ` doc for each release of Python 3 +* the `Porting to Python 3`_ book (which is free online) +* the handy `cheat sheet`_ from the Python-Future project. Update your code ---------------- -Once you feel like you know what is different in Python 3 compared to Python 2, -it's time to update your code! You have a choice between two tools in porting -your code automatically: Futurize_ and Modernize_. Which tool you choose will -depend on how much like Python 3 you want your code to be. Futurize_ does its -best to make Python 3 idioms and practices exist in Python 2, e.g. backporting -the ``bytes`` type from Python 3 so that you have semantic parity between the -major versions of Python. Modernize_, -on the other hand, is more conservative and targets a Python 2/3 subset of -Python, directly relying on six_ to help provide compatibility. As Python 3 is -the future, it might be best to consider Futurize to begin adjusting to any new -practices that Python 3 introduces which you are not accustomed to yet. - -Regardless of which tool you choose, they will update your code to run under -Python 3 while staying compatible with the version of Python 2 you started with. -Depending on how conservative you want to be, you may want to run the tool over -your test suite first and visually inspect the diff to make sure the -transformation is accurate. After you have transformed your test suite and -verified that all the tests still pass as expected, then you can transform your -application code knowing that any tests which fail is a translation failure. +There are tools available that can port your code automatically. + +Futurize_ does its best to make Python 3 idioms and practices exist in Python +2, e.g. backporting the ``bytes`` type from Python 3 so that you have +semantic parity between the major versions of Python. This is the better +approach for most cases. + +Modernize_, on the other hand, is more conservative and targets a Python 2/3 +subset of Python, directly relying on six_ to help provide compatibility. + +A good approach is to run the tool over your test suite first and visually +inspect the diff to make sure the transformation is accurate. After you have +transformed your test suite and verified that all the tests still pass as +expected, then you can transform your application code knowing that any tests +which fail is a translation failure. Unfortunately the tools can't automate everything to make your code work under -Python 3 and so there are a handful of things you will need to update manually -to get full Python 3 support (which of these steps are necessary vary between -the tools). Read the documentation for the tool you choose to use to see what it -fixes by default and what it can do optionally to know what will (not) be fixed -for you and what you may have to fix on your own (e.g. using ``io.open()`` over -the built-in ``open()`` function is off by default in Modernize). Luckily, -though, there are only a couple of things to watch out for which can be -considered large issues that may be hard to debug if not watched for. +Python 3, and you will also need to read the tools' documentation in case some +options you need are turned off by default. +Key issues to be aware of and check for: Division ++++++++ -In Python 3, ``5 / 2 == 2.5`` and not ``2``; all division between ``int`` values -result in a ``float``. This change has actually been planned since Python 2.2 -which was released in 2002. Since then users have been encouraged to add -``from __future__ import division`` to any and all files which use the ``/`` and -``//`` operators or to be running the interpreter with the ``-Q`` flag. If you -have not been doing this then you will need to go through your code and do two -things: +In Python 3, ``5 / 2 == 2.5`` and not ``2`` as it was in Python 2; all +division between ``int`` values result in a ``float``. This change has +actually been planned since Python 2.2 which was released in 2002. Since then +users have been encouraged to add ``from __future__ import division`` to any +and all files which use the ``/`` and ``//`` operators or to be running the +interpreter with the ``-Q`` flag. If you have not been doing this then you +will need to go through your code and do two things: #. Add ``from __future__ import division`` to your files #. Update any division operator as necessary to either use ``//`` to use floor @@ -197,30 +178,29 @@ specific type. This complicated the situation especially for anyone supporting multiple languages as APIs wouldn't bother explicitly supporting ``unicode`` when they claimed text data support. -To make the distinction between text and binary data clearer and more -pronounced, Python 3 did what most languages created in the age of the internet -have done and made text and binary data distinct types that cannot blindly be -mixed together (Python predates widespread access to the internet). For any code -that deals only with text or only binary data, this separation doesn't pose an -issue. But for code that has to deal with both, it does mean you might have to -now care about when you are using text compared to binary data, which is why -this cannot be entirely automated. - -To start, you will need to decide which APIs take text and which take binary -(it is **highly** recommended you don't design APIs that can take both due to -the difficulty of keeping the code working; as stated earlier it is difficult to -do well). In Python 2 this means making sure the APIs that take text can work -with ``unicode`` and those that work with binary data work with the -``bytes`` type from Python 3 (which is a subset of ``str`` in Python 2 and acts -as an alias for ``bytes`` type in Python 2). Usually the biggest issue is -realizing which methods exist on which types in Python 2 & 3 simultaneously -(for text that's ``unicode`` in Python 2 and ``str`` in Python 3, for binary -that's ``str``/``bytes`` in Python 2 and ``bytes`` in Python 3). The following -table lists the **unique** methods of each data type across Python 2 & 3 -(e.g., the ``decode()`` method is usable on the equivalent binary data type in -either Python 2 or 3, but it can't be used by the textual data type consistently -between Python 2 and 3 because ``str`` in Python 3 doesn't have the method). Do -note that as of Python 3.5 the ``__mod__`` method was added to the bytes type. +Python 3 made text and binary data distinct types that cannot simply be mixed +together. For any code that deals only with text or only binary data, this +separation doesn't pose an issue. But for code that has to deal with both, it +does mean you might have to now care about when you are using text compared +to binary data, which is why this cannot be entirely automated. + +Decide which APIs take text and which take binary (it is **highly** recommended +you don't design APIs that can take both due to the difficulty of keeping the +code working; as stated earlier it is difficult to do well). In Python 2 this +means making sure the APIs that take text can work with ``unicode`` and those +that work with binary data work with the ``bytes`` type from Python 3 +(which is a subset of ``str`` in Python 2 and acts as an alias for ``bytes`` +type in Python 2). Usually the biggest issue is realizing which methods exist +on which types in Python 2 and 3 simultaneously (for text that's ``unicode`` +in Python 2 and ``str`` in Python 3, for binary that's ``str``/``bytes`` in +Python 2 and ``bytes`` in Python 3). + +The following table lists the **unique** methods of each data type across +Python 2 and 3 (e.g., the ``decode()`` method is usable on the equivalent binary +data type in either Python 2 or 3, but it can't be used by the textual data +type consistently between Python 2 and 3 because ``str`` in Python 3 doesn't +have the method). Do note that as of Python 3.5 the ``__mod__`` method was +added to the bytes type. ======================== ===================== **Text data** **Binary data** @@ -246,12 +226,11 @@ having to keep track of what type of data you are working with. The next issue is making sure you know whether the string literals in your code represent text or binary data. You should add a ``b`` prefix to any literal that presents binary data. For text you should add a ``u`` prefix to -the text literal. (there is a :mod:`__future__` import to force all unspecified +the text literal. (There is a :mod:`__future__` import to force all unspecified literals to be Unicode, but usage has shown it isn't as effective as adding a ``b`` or ``u`` prefix to all literals explicitly) -As part of this dichotomy you also need to be careful about opening files. -Unless you have been working on Windows, there is a chance you have not always +You also need to be careful about opening files. Possibly you have not always bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for binary reading). Under Python 3, binary files and text files are clearly distinct and mutually incompatible; see the :mod:`io` module for details. @@ -265,7 +244,7 @@ outdated practice of using :func:`codecs.open` as that's only necessary for keeping compatibility with Python 2.5. The constructors of both ``str`` and ``bytes`` have different semantics for the -same arguments between Python 2 & 3. Passing an integer to ``bytes`` in Python 2 +same arguments between Python 2 and 3. Passing an integer to ``bytes`` in Python 2 will give you the string representation of the integer: ``bytes(3) == '3'``. But in Python 3, an integer argument to ``bytes`` will give you a bytes object as long as the integer specified, filled with null bytes: @@ -400,7 +379,7 @@ Use continuous integration to stay compatible --------------------------------------------- Once you are able to fully run under Python 3 you will want to make sure your -code always works under both Python 2 & 3. Probably the best tool for running +code always works under both Python 2 and 3. Probably the best tool for running your tests under multiple Python interpreters is tox_. You can then integrate tox with your continuous integration system so that you never accidentally break Python 2 or 3 support. @@ -413,16 +392,11 @@ separation of text/binary data handling or indexing on bytes you wouldn't easily find the mistake. This flag will raise an exception when these kinds of comparisons occur, making the mistake much easier to track down. -And that's mostly it! At this point your code base is compatible with both -Python 2 and 3 simultaneously. Your testing will also be set up so that you -don't accidentally break Python 2 or 3 compatibility regardless of which version -you typically run your tests under while developing. - Consider using optional static type checking -------------------------------------------- -Another way to help port your code is to use a static type checker like +Another way to help port your code is to use a :term:`static type checker` like mypy_ or pytype_ on your code. These tools can be used to analyze your code as if it's being run under Python 2, then you can run the tool a second time as if your code is running under Python 3. By running a static type checker twice like diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 655df59e27b641..5e2f9a9d1837fe 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -245,6 +245,9 @@ You can omit either *m* or *n*; in that case, a reasonable value is assumed for the missing value. Omitting *m* is interpreted as a lower limit of 0, while omitting *n* results in an upper bound of infinity. +The simplest case ``{m}`` matches the preceding item exactly *m* times. +For example, ``a/{2}b`` will only match ``'a//b'``. + Readers of a reductionist bent may notice that the three other quantifiers can all be expressed using this notation. ``{0,}`` is the same as ``*``, ``{1,}`` is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use @@ -518,6 +521,8 @@ cache. Compilation Flags ----------------- +.. currentmodule:: re + Compilation flags let you modify some aspects of how regular expressions work. Flags are available in the :mod:`re` module under two names, a long name such as :const:`IGNORECASE` and a short, one-letter form such as :const:`I`. (If you're diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index decce12bf3faf6..38dd09f0a721d2 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -273,7 +273,7 @@ Odds and Ends * The sort routines use ``<`` when making comparisons between two objects. So, it is easy to add a standard sort order to a class by - defining an :meth:`__lt__` method: + defining an :meth:`~object.__lt__` method: .. doctest:: @@ -281,8 +281,8 @@ Odds and Ends >>> sorted(student_objects) [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - However, note that ``<`` can fall back to using :meth:`__gt__` if - :meth:`__lt__` is not implemented (see :func:`object.__lt__`). + However, note that ``<`` can fall back to using :meth:`~object.__gt__` if + :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__`). * Key functions need not depend directly on the objects being sorted. A key function can also access external resources. For instance, if the student grades diff --git a/Doc/howto/timerfd.rst b/Doc/howto/timerfd.rst new file mode 100644 index 00000000000000..98f0294f9d082d --- /dev/null +++ b/Doc/howto/timerfd.rst @@ -0,0 +1,230 @@ +.. _timerfd-howto: + +***************************** + timer file descriptor HOWTO +***************************** + +:Release: 1.13 + +This HOWTO discusses Python's support for the linux timer file descriptor. + + +Examples +======== + +The following example shows how to use a timer file descriptor +to execute a function twice a second: + +.. code-block:: python + + # Practical scripts should use really use a non-blocking timer, + # we use a blocking timer here for simplicity. + import os, time + + # Create the timer file descriptor + fd = os.timerfd_create(time.CLOCK_REALTIME) + + # Start the timer in 1 second, with an interval of half a second + os.timerfd_settime(fd, initial=1, interval=0.5) + + try: + # Process timer events four times. + for _ in range(4): + # read() will block until the timer expires + _ = os.read(fd, 8) + print("Timer expired") + finally: + # Remember to close the timer file descriptor! + os.close(fd) + +To avoid the precision loss caused by the :class:`float` type, +timer file descriptors allow specifying initial expiration and interval +in integer nanoseconds with ``_ns`` variants of the functions. + +This example shows how :func:`~select.epoll` can be used with timer file +descriptors to wait until the file descriptor is ready for reading: + +.. code-block:: python + + import os, time, select, socket, sys + + # Create an epoll object + ep = select.epoll() + + # In this example, use loopback address to send "stop" command to the server. + # + # $ telnet 127.0.0.1 1234 + # Trying 127.0.0.1... + # Connected to 127.0.0.1. + # Escape character is '^]'. + # stop + # Connection closed by foreign host. + # + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 1234)) + sock.setblocking(False) + sock.listen(1) + ep.register(sock, select.EPOLLIN) + + # Create timer file descriptors in non-blocking mode. + num = 3 + fds = [] + for _ in range(num): + fd = os.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + fds.append(fd) + # Register the timer file descriptor for read events + ep.register(fd, select.EPOLLIN) + + # Start the timer with os.timerfd_settime_ns() in nanoseconds. + # Timer 1 fires every 0.25 seconds; timer 2 every 0.5 seconds; etc + for i, fd in enumerate(fds, start=1): + one_sec_in_nsec = 10**9 + i = i * one_sec_in_nsec + os.timerfd_settime_ns(fd, initial=i//4, interval=i//4) + + timeout = 3 + try: + conn = None + is_active = True + while is_active: + # Wait for the timer to expire for 3 seconds. + # epoll.poll() returns a list of (fd, event) pairs. + # fd is a file descriptor. + # sock and conn[=returned value of socket.accept()] are socket objects, not file descriptors. + # So use sock.fileno() and conn.fileno() to get the file descriptors. + events = ep.poll(timeout) + + # If more than one timer file descriptors are ready for reading at once, + # epoll.poll() returns a list of (fd, event) pairs. + # + # In this example settings, + # 1st timer fires every 0.25 seconds in 0.25 seconds. (0.25, 0.5, 0.75, 1.0, ...) + # 2nd timer every 0.5 seconds in 0.5 seconds. (0.5, 1.0, 1.5, 2.0, ...) + # 3rd timer every 0.75 seconds in 0.75 seconds. (0.75, 1.5, 2.25, 3.0, ...) + # + # In 0.25 seconds, only 1st timer fires. + # In 0.5 seconds, 1st timer and 2nd timer fires at once. + # In 0.75 seconds, 1st timer and 3rd timer fires at once. + # In 1.5 seconds, 1st timer, 2nd timer and 3rd timer fires at once. + # + # If a timer file descriptor is signaled more than once since + # the last os.read() call, os.read() returns the nubmer of signaled + # as host order of class bytes. + print(f"Signaled events={events}") + for fd, event in events: + if event & select.EPOLLIN: + if fd == sock.fileno(): + # Check if there is a connection request. + print(f"Accepting connection {fd}") + conn, addr = sock.accept() + conn.setblocking(False) + print(f"Accepted connection {conn} from {addr}") + ep.register(conn, select.EPOLLIN) + elif conn and fd == conn.fileno(): + # Check if there is data to read. + print(f"Reading data {fd}") + data = conn.recv(1024) + if data: + # You should catch UnicodeDecodeError exception for safety. + cmd = data.decode() + if cmd.startswith("stop"): + print(f"Stopping server") + is_active = False + else: + print(f"Unknown command: {cmd}") + else: + # No more data, close connection + print(f"Closing connection {fd}") + ep.unregister(conn) + conn.close() + conn = None + elif fd in fds: + print(f"Reading timer {fd}") + count = int.from_bytes(os.read(fd, 8), byteorder=sys.byteorder) + print(f"Timer {fds.index(fd) + 1} expired {count} times") + else: + print(f"Unknown file descriptor {fd}") + finally: + for fd in fds: + ep.unregister(fd) + os.close(fd) + ep.close() + +This example shows how :func:`~select.select` can be used with timer file +descriptors to wait until the file descriptor is ready for reading: + +.. code-block:: python + + import os, time, select, socket, sys + + # In this example, use loopback address to send "stop" command to the server. + # + # $ telnet 127.0.0.1 1234 + # Trying 127.0.0.1... + # Connected to 127.0.0.1. + # Escape character is '^]'. + # stop + # Connection closed by foreign host. + # + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 1234)) + sock.setblocking(False) + sock.listen(1) + + # Create timer file descriptors in non-blocking mode. + num = 3 + fds = [os.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + for _ in range(num)] + select_fds = fds + [sock] + + # Start the timers with os.timerfd_settime() in seconds. + # Timer 1 fires every 0.25 seconds; timer 2 every 0.5 seconds; etc + for i, fd in enumerate(fds, start=1): + os.timerfd_settime(fd, initial=i/4, interval=i/4) + + timeout = 3 + try: + conn = None + is_active = True + while is_active: + # Wait for the timer to expire for 3 seconds. + # select.select() returns a list of file descriptors or objects. + rfd, wfd, xfd = select.select(select_fds, select_fds, select_fds, timeout) + for fd in rfd: + if fd == sock: + # Check if there is a connection request. + print(f"Accepting connection {fd}") + conn, addr = sock.accept() + conn.setblocking(False) + print(f"Accepted connection {conn} from {addr}") + select_fds.append(conn) + elif conn and fd == conn: + # Check if there is data to read. + print(f"Reading data {fd}") + data = conn.recv(1024) + if data: + # You should catch UnicodeDecodeError exception for safety. + cmd = data.decode() + if cmd.startswith("stop"): + print(f"Stopping server") + is_active = False + else: + print(f"Unknown command: {cmd}") + else: + # No more data, close connection + print(f"Closing connection {fd}") + select_fds.remove(conn) + conn.close() + conn = None + elif fd in fds: + print(f"Reading timer {fd}") + count = int.from_bytes(os.read(fd, 8), byteorder=sys.byteorder) + print(f"Timer {fds.index(fd) + 1} expired {count} times") + else: + print(f"Unknown file descriptor {fd}") + finally: + for fd in fds: + os.close(fd) + sock.close() + sock = None + diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index b0faa68d240896..254fe729355353 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -424,8 +424,8 @@ lowercase letters 'ss'. A second tool is the :mod:`unicodedata` module's :func:`~unicodedata.normalize` function that converts strings to one -of several normal forms, where letters followed by a combining -character are replaced with single characters. :func:`normalize` can +of several normal forms, where letters followed by a combining character are +replaced with single characters. :func:`~unicodedata.normalize` can be used to perform string comparisons that won't falsely report inequality if two strings use combining characters differently: @@ -474,8 +474,8 @@ The Unicode Standard also specifies how to do caseless comparisons:: print(compare_caseless(single_char, multiple_chars)) -This will print ``True``. (Why is :func:`NFD` invoked twice? Because -there are a few characters that make :meth:`casefold` return a +This will print ``True``. (Why is :func:`!NFD` invoked twice? Because +there are a few characters that make :meth:`~str.casefold` return a non-normalized string, so the result needs to be normalized again. See section 3.13 of the Unicode Standard for a discussion and an example.) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 61ba6bd7224fcc..570435d48866d3 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -6,13 +6,6 @@ :Author: `Michael Foord `_ -.. note:: - - There is a French translation of an earlier revision of this - HOWTO, available at `urllib2 - Le Manuel manquant - `_. - - Introduction ============ @@ -201,11 +194,11 @@ which comes after we have a look at what happens when things go wrong. Handling Exceptions =================== -*urlopen* raises :exc:`URLError` when it cannot handle a response (though as +*urlopen* raises :exc:`~urllib.error.URLError` when it cannot handle a response (though as usual with Python APIs, built-in exceptions such as :exc:`ValueError`, :exc:`TypeError` etc. may also be raised). -:exc:`HTTPError` is the subclass of :exc:`URLError` raised in the specific case of +:exc:`~urllib.error.HTTPError` is the subclass of :exc:`~urllib.error.URLError` raised in the specific case of HTTP URLs. The exception classes are exported from the :mod:`urllib.error` module. @@ -236,12 +229,12 @@ the status code indicates that the server is unable to fulfil the request. The default handlers will handle some of these responses for you (for example, if the response is a "redirection" that requests the client fetch the document from a different URL, urllib will handle that for you). For those it can't handle, -urlopen will raise an :exc:`HTTPError`. Typical errors include '404' (page not +urlopen will raise an :exc:`~urllib.error.HTTPError`. Typical errors include '404' (page not found), '403' (request forbidden), and '401' (authentication required). See section 10 of :rfc:`2616` for a reference on all the HTTP error codes. -The :exc:`HTTPError` instance raised will have an integer 'code' attribute, which +The :exc:`~urllib.error.HTTPError` instance raised will have an integer 'code' attribute, which corresponds to the error sent by the server. Error Codes @@ -324,7 +317,7 @@ dictionary is reproduced here for convenience :: } When an error is raised the server responds by returning an HTTP error code -*and* an error page. You can use the :exc:`HTTPError` instance as a response on the +*and* an error page. You can use the :exc:`~urllib.error.HTTPError` instance as a response on the page returned. This means that as well as the code attribute, it also has read, geturl, and info, methods as returned by the ``urllib.response`` module:: @@ -345,7 +338,7 @@ geturl, and info, methods as returned by the ``urllib.response`` module:: Wrapping it Up -------------- -So if you want to be prepared for :exc:`HTTPError` *or* :exc:`URLError` there are two +So if you want to be prepared for :exc:`~urllib.error.HTTPError` *or* :exc:`~urllib.error.URLError` there are two basic approaches. I prefer the second approach. Number 1 @@ -372,7 +365,7 @@ Number 1 .. note:: The ``except HTTPError`` *must* come first, otherwise ``except URLError`` - will *also* catch an :exc:`HTTPError`. + will *also* catch an :exc:`~urllib.error.HTTPError`. Number 2 ~~~~~~~~ @@ -398,7 +391,7 @@ Number 2 info and geturl =============== -The response returned by urlopen (or the :exc:`HTTPError` instance) has two +The response returned by urlopen (or the :exc:`~urllib.error.HTTPError` instance) has two useful methods :meth:`info` and :meth:`geturl` and is defined in the module :mod:`urllib.response`.. diff --git a/Doc/includes/email-alternative.py b/Doc/includes/email-alternative.py index df7ca6f3faa332..26b302b495c7ac 100644 --- a/Doc/includes/email-alternative.py +++ b/Doc/includes/email-alternative.py @@ -8,14 +8,14 @@ # Create the base text message. msg = EmailMessage() -msg['Subject'] = "Ayons asperges pour le déjeuner" +msg['Subject'] = "Pourquoi pas des asperges pour ce midi ?" msg['From'] = Address("Pepé Le Pew", "pepe", "example.com") msg['To'] = (Address("Penelope Pussycat", "penelope", "example.com"), Address("Fabrette Pussycat", "fabrette", "example.com")) msg.set_content("""\ Salut! -Cela ressemble à un excellent recipie[1] déjeuner. +Cette recette [1] sera sûrement un très bon repas. [1] http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718 @@ -31,10 +31,10 @@

Salut!

-

Cela ressemble à un excellent +

diff --git a/Doc/includes/custom.c b/Doc/includes/newtypes/custom.c similarity index 86% rename from Doc/includes/custom.c rename to Doc/includes/newtypes/custom.c index 9cfba50ace25db..5253f879360210 100644 --- a/Doc/includes/custom.c +++ b/Doc/includes/newtypes/custom.c @@ -34,9 +34,7 @@ PyInit_custom(void) if (m == NULL) return NULL; - Py_INCREF(&CustomType); - if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(&CustomType); + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { Py_DECREF(m); return NULL; } diff --git a/Doc/includes/custom2.c b/Doc/includes/newtypes/custom2.c similarity index 100% rename from Doc/includes/custom2.c rename to Doc/includes/newtypes/custom2.c diff --git a/Doc/includes/custom3.c b/Doc/includes/newtypes/custom3.c similarity index 100% rename from Doc/includes/custom3.c rename to Doc/includes/newtypes/custom3.c diff --git a/Doc/includes/custom4.c b/Doc/includes/newtypes/custom4.c similarity index 100% rename from Doc/includes/custom4.c rename to Doc/includes/newtypes/custom4.c diff --git a/Doc/includes/newtypes/pyproject.toml b/Doc/includes/newtypes/pyproject.toml new file mode 100644 index 00000000000000..ea7937a3171473 --- /dev/null +++ b/Doc/includes/newtypes/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "custom" +version = "1" diff --git a/Doc/includes/newtypes/setup.py b/Doc/includes/newtypes/setup.py new file mode 100644 index 00000000000000..67f83673bcc657 --- /dev/null +++ b/Doc/includes/newtypes/setup.py @@ -0,0 +1,8 @@ +from setuptools import Extension, setup +setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + Extension("custom3", ["custom3.c"]), + Extension("custom4", ["custom4.c"]), + Extension("sublist", ["sublist.c"]), +]) diff --git a/Doc/includes/sublist.c b/Doc/includes/newtypes/sublist.c similarity index 91% rename from Doc/includes/sublist.c rename to Doc/includes/newtypes/sublist.c index b36dadf07eae87..d8aba463f30ba2 100644 --- a/Doc/includes/sublist.c +++ b/Doc/includes/newtypes/sublist.c @@ -58,9 +58,7 @@ PyInit_sublist(void) if (m == NULL) return NULL; - Py_INCREF(&SubListType); - if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(&SubListType); + if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { Py_DECREF(m); return NULL; } diff --git a/Doc/includes/test.py b/Doc/includes/newtypes/test.py similarity index 94% rename from Doc/includes/test.py rename to Doc/includes/newtypes/test.py index 09ebe3fec0bdbe..55a5cf9f68b94a 100644 --- a/Doc/includes/test.py +++ b/Doc/includes/newtypes/test.py @@ -187,13 +187,6 @@ >>> gc.enable() """ -import os -import sys -from distutils.util import get_platform -PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2]) -src = os.path.join("build", "lib.%s" % PLAT_SPEC) -sys.path.append(src) - if __name__ == "__main__": import doctest, __main__ doctest.testmod(__main__) diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py deleted file mode 100644 index a38a39de3e7c86..00000000000000 --- a/Doc/includes/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from distutils.core import setup, Extension -setup(name="noddy", version="1.0", - ext_modules=[ - Extension("noddy", ["noddy.c"]), - Extension("noddy2", ["noddy2.c"]), - Extension("noddy3", ["noddy3.c"]), - Extension("noddy4", ["noddy4.c"]), - Extension("shoddy", ["shoddy.c"]), - ]) diff --git a/Doc/includes/turtle-star.py b/Doc/includes/turtle-star.py deleted file mode 100644 index 1a5db761b32385..00000000000000 --- a/Doc/includes/turtle-star.py +++ /dev/null @@ -1,10 +0,0 @@ -from turtle import * -color('red', 'yellow') -begin_fill() -while True: - forward(200) - left(170) - if abs(pos()) < 1: - break -end_fill() -done() diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index f0ad1e47cb0d86..ec939c28831c33 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -82,5 +82,5 @@ typedef struct _typeobject { vectorcallfunc tp_vectorcall; /* bitset of which type-watchers care about this type */ - char tp_watched; + unsigned char tp_watched; } PyTypeObject; diff --git a/Doc/install/index.rst b/Doc/install/index.rst deleted file mode 100644 index ab581d785ef7f0..00000000000000 --- a/Doc/install/index.rst +++ /dev/null @@ -1,1081 +0,0 @@ -.. highlight:: none - -.. _install-index: - -******************************************** - Installing Python Modules (Legacy version) -******************************************** - -:Author: Greg Ward - -.. TODO: Fill in XXX comments - -.. note:: - - The entire ``distutils`` package has been deprecated and will be - removed in Python 3.12. This documentation is retained as a - reference only, and will be removed with the package. See the - :ref:`What's New ` entry for more information. - -.. seealso:: - - :ref:`installing-index` - The up to date module installation documentation. For regular Python - usage, you almost certainly want that document rather than this one. - -.. note:: - - This document is being retained solely until the ``setuptools`` documentation - at https://setuptools.readthedocs.io/en/latest/setuptools.html - independently covers all of the relevant information currently included here. - -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ - in the Python Packaging User Guide for more information. - - -.. _inst-intro: - - -Introduction -============ - -In Python 2.0, the ``distutils`` API was first added to the standard library. -This provided Linux distro maintainers with a standard way of converting -Python projects into Linux distro packages, and system administrators with a -standard way of installing them directly onto target systems. - -In the many years since Python 2.0 was released, tightly coupling the build -system and package installer to the language runtime release cycle has turned -out to be problematic, and it is now recommended that projects use the -``pip`` package installer and the ``setuptools`` build system, rather than -using ``distutils`` directly. - -See :ref:`installing-index` and :ref:`distributing-index` for more details. - -This legacy documentation is being retained only until we're confident that the -``setuptools`` documentation covers everything needed. - -.. _inst-new-standard: - -Distutils based source distributions ------------------------------------- - -If you download a module source distribution, you can tell pretty quickly if it -was packaged and distributed in the standard way, i.e. using the Distutils. -First, the distribution's name and version number will be featured prominently -in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or -:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly named -directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the -distribution will contain a setup script :file:`setup.py`, and a file named -:file:`README.txt` or possibly just :file:`README`, which should explain that -building and installing the module distribution is a simple matter of running -one command from a terminal:: - - python setup.py install - -For Windows, this command should be run from a command prompt window -(:menuselection:`Start --> Accessories`):: - - setup.py install - -If all these things are true, then you already know how to build and install the -modules you've just downloaded: Run the command above. Unless you need to -install things in a non-standard way or customize the build process, you don't -really need this manual. Or rather, the above command is everything you need to -get out of this manual. - - -.. _inst-standard-install: - -Standard Build and Install -========================== - -As described in section :ref:`inst-new-standard`, building and installing a module -distribution using the Distutils is usually one simple command to run from a -terminal:: - - python setup.py install - - -.. _inst-platform-variations: - -Platform variations -------------------- - -You should always run the setup command from the distribution root directory, -i.e. the top-level subdirectory that the module source distribution unpacks -into. For example, if you've just downloaded a module source distribution -:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: - - gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 - cd foo-1.0 - python setup.py install - -On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the -archive file to :file:`C:\\Temp`, then it would unpack into -:file:`C:\\Temp\\foo-1.0`; you can use either an archive manipulator with a -graphical user interface (such as WinZip) or a command-line tool (such as -:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a -command prompt window and run:: - - cd c:\Temp\foo-1.0 - python setup.py install - - -.. _inst-splitting-up: - -Splitting the job up --------------------- - -Running ``setup.py install`` builds and installs all modules in one run. If you -prefer to work incrementally---especially useful if you want to customize the -build process, or if things are going wrong---you can use the setup script to do -one thing at a time. This is particularly helpful when the build and install -will be done by different users---for example, you might want to build a module -distribution and hand it off to a system administrator for installation (or do -it yourself, with super-user privileges). - -For example, you can build everything in one step, and then install everything -in a second step, by invoking the setup script twice:: - - python setup.py build - python setup.py install - -If you do this, you will notice that running the :command:`install` command -first runs the :command:`build` command, which---in this case---quickly notices -that it has nothing to do, since everything in the :file:`build` directory is -up-to-date. - -You may not need this ability to break things down often if all you do is -install modules downloaded off the 'net, but it's very handy for more advanced -tasks. If you get into distributing your own Python modules and extensions, -you'll run lots of individual Distutils commands on their own. - - -.. _inst-how-build-works: - -How building works ------------------- - -As implied above, the :command:`build` command is responsible for putting the -files to install into a *build directory*. By default, this is :file:`build` -under the distribution root; if you're excessively concerned with speed, or want -to keep the source tree pristine, you can change the build directory with the -:option:`!--build-base` option. For example:: - - python setup.py build --build-base=/path/to/pybuild/foo-1.0 - -(Or you could do this permanently with a directive in your system or personal -Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this -isn't necessary. - -The default layout for the build tree is as follows:: - - --- build/ --- lib/ - or - --- build/ --- lib./ - temp./ - -where ```` expands to a brief description of the current OS/hardware -platform and Python version. The first form, with just a :file:`lib` directory, -is used for "pure module distributions"---that is, module distributions that -include only pure Python modules. If a module distribution contains any -extensions (modules written in C/C++), then the second form, with two ```` -directories, is used. In that case, the :file:`temp.{plat}` directory holds -temporary files generated by the compile/link process that don't actually get -installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory -contains all Python modules (pure Python and extensions) that will be installed. - -In the future, more directories will be added to handle Python scripts, -documentation, binary executables, and whatever else is needed to handle the job -of installing Python modules and applications. - - -.. _inst-how-install-works: - -How installation works ----------------------- - -After the :command:`build` command runs (whether you run it explicitly, or the -:command:`install` command does it for you), the work of the :command:`install` -command is relatively simple: all it has to do is copy everything under -:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation -directory. - -If you don't choose an installation directory---i.e., if you just run ``setup.py -install``\ ---then the :command:`install` command installs to the standard -location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and macOS, which is also -Unix-based), it also depends on whether the module distribution being installed -is pure Python or contains extensions ("non-pure"): - -.. tabularcolumns:: |l|l|l|l| - -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Platform | Standard installation location | Default value | Notes | -+=================+=====================================================+==================================================+=======+ -| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ - -Notes: - -(1) - Most Linux distributions include Python as a standard part of the system, so - :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on - Linux. If you build Python yourself on Linux (or any Unix-like system), the - default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. - -(2) - The default installation directory on Windows was :file:`C:\\Program - Files\\Python` under Python 1.6a1, 1.5.2, and earlier. - -:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python -is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and macOS. You -can find out what your Python installation uses for :file:`{prefix}` and -:file:`{exec-prefix}` by running Python in interactive mode and typing a few -simple commands. Under Unix, just type ``python`` at the shell prompt. Under -Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> -Python (command line)`. Once the interpreter is started, you type Python code -at the prompt. For example, on my Linux system, I type the three Python -statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`: - -.. code-block:: pycon - - Python 2.4 (#26, Aug 7 2004, 17:19:02) - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.prefix - '/usr' - >>> sys.exec_prefix - '/usr' - -A few other placeholders are used in this document: :file:`{X.Y}` stands for the -version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by -the value of :data:`sys.abiflags` or the empty string for platforms which don't -define ABI flags; :file:`{distname}` will be replaced by the name of the module -distribution being installed. Dots and capitalization are important in the -paths; for example, a value that uses ``python3.2`` on UNIX will typically use -``Python32`` on Windows. - -If you don't want to install modules to the standard location, or if you don't -have permission to write there, then you need to read about alternate -installations in section :ref:`inst-alt-install`. If you want to customize your -installation directories more heavily, see section :ref:`inst-custom-install` on -custom installations. - - -.. _inst-alt-install: - -Alternate Installation -====================== - -Often, it is necessary or desirable to install modules to a location other than -the standard location for third-party Python modules. For example, on a Unix -system you might not have permission to write to the standard third-party module -directory. Or you might wish to try out a module before making it a standard -part of your local Python installation. This is especially true when upgrading -a distribution already present: you want to make sure your existing base of -scripts still works with the new version before actually upgrading. - -The Distutils :command:`install` command is designed to make installing module -distributions to an alternate location simple and painless. The basic idea is -that you supply a base directory for the installation, and the -:command:`install` command picks a set of directories (called an *installation -scheme*) under this base directory in which to install files. The details -differ across platforms, so read whichever of the following sections applies to -you. - -Note that the various alternate installation schemes are mutually exclusive: you -can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or -``--install-base`` and ``--install-platbase``, but you can't mix from these -groups. - - -.. _inst-alt-install-user: - -Alternate installation: the user scheme ---------------------------------------- - -This scheme is designed to be the most convenient solution for users that don't -have write permission to the global site-packages directory or don't want to -install into it. It is enabled with a simple option:: - - python setup.py install --user - -Files will be installed into subdirectories of :data:`site.USER_BASE` (written -as :file:`{userbase}` hereafter). This scheme installs pure Python modules and -extension modules in the same location (also known as :data:`site.USER_SITE`). -Here are the values for UNIX, including macOS: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}/lib/python{X.Y}/site-packages` -scripts :file:`{userbase}/bin` -data :file:`{userbase}` -C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}` -=============== =========================================================== - -And here are the values used on Windows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}\\Python{XY}\\site-packages` -scripts :file:`{userbase}\\Python{XY}\\Scripts` -data :file:`{userbase}` -C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}` -=============== =========================================================== - -The advantage of using this scheme compared to the other ones described below is -that the user site-packages directory is under normal conditions always included -in :data:`sys.path` (see :mod:`site` for more information), which means that -there is no additional step to perform after running the :file:`setup.py` script -to finalize the installation. - -The :command:`build_ext` command also has a ``--user`` option to add -:file:`{userbase}/include` to the compiler search path for header files and -:file:`{userbase}/lib` to the compiler search path for libraries as well as to -the runtime search path for shared C libraries (rpath). - - -.. _inst-alt-install-home: - -Alternate installation: the home scheme ---------------------------------------- - -The idea behind the "home scheme" is that you build and maintain a personal -stash of Python modules. This scheme's name is derived from the idea of a -"home" directory on Unix, since it's not unusual for a Unix user to make their -home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. -This scheme can be used by anyone, regardless of the operating system they -are installing for. - -Installing a new module distribution is as simple as :: - - python setup.py install --home= - -where you can supply any directory you like for the :option:`!--home` option. On -Unix, lazy typists can just type a tilde (``~``); the :command:`install` command -will expand this to your home directory:: - - python setup.py install --home=~ - -To make Python find the distributions installed with this scheme, you may have -to :ref:`modify Python's search path ` or edit -:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit -:data:`sys.path`. - -The :option:`!--home` option defines the installation base directory. Files are -installed to the following directories under the installation base as follows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{home}/lib/python` -scripts :file:`{home}/bin` -data :file:`{home}` -C headers :file:`{home}/include/python/{distname}` -=============== =========================================================== - -(Mentally replace slashes with backslashes if you're on Windows.) - - -.. _inst-alt-install-prefix-unix: - -Alternate installation: Unix (the prefix scheme) ------------------------------------------------- - -The "prefix scheme" is useful when you wish to use one Python installation to -perform the build/install (i.e., to run the setup script), but install modules -into the third-party module directory of a different Python installation (or -something that looks like a different Python installation). If this sounds a -trifle unusual, it is---that's why the user and home schemes come before. However, -there are at least two known cases where the prefix scheme will be useful. - -First, consider that many Linux distributions put Python in :file:`/usr`, rather -than the more traditional :file:`/usr/local`. This is entirely appropriate, -since in those cases Python is part of "the system" rather than a local add-on. -However, if you are installing Python modules from source, you probably want -them to go in :file:`/usr/local/lib/python2.{X}` rather than -:file:`/usr/lib/python2.{X}`. This can be done with :: - - /usr/bin/python setup.py install --prefix=/usr/local - -Another possibility is a network filesystem where the name used to write to a -remote directory is different from the name used to read it: for example, the -Python interpreter accessed as :file:`/usr/local/bin/python` might search for -modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to -be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could -be done with :: - - /usr/local/bin/python setup.py install --prefix=/mnt/@server/export - -In either case, the :option:`!--prefix` option defines the installation base, and -the :option:`!--exec-prefix` option defines the platform-specific installation -base, which is used for platform-specific files. (Currently, this just means -non-pure module distributions, but could be expanded to C libraries, binary -executables, etc.) If :option:`!--exec-prefix` is not supplied, it defaults to -:option:`!--prefix`. Files are installed as follows: - -================= ========================================================== -Type of file Installation directory -================= ========================================================== -Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` -extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` -scripts :file:`{prefix}/bin` -data :file:`{prefix}` -C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}` -================= ========================================================== - -There is no requirement that :option:`!--prefix` or :option:`!--exec-prefix` -actually point to an alternate Python installation; if the directories listed -above do not already exist, they are created at installation time. - -Incidentally, the real reason the prefix scheme is important is simply that a -standard Unix installation uses the prefix scheme, but with :option:`!--prefix` -and :option:`!--exec-prefix` supplied by Python itself as ``sys.prefix`` and -``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, -but every time you run ``python setup.py install`` without any other options, -you're using it. - -Note that installing extensions to an alternate Python installation has no -effect on how those extensions are built: in particular, the Python header files -(:file:`Python.h` and friends) installed with the Python interpreter used to run -the setup script will be used in compiling extensions. It is your -responsibility to ensure that the interpreter used to run extensions installed -in this way is compatible with the interpreter used to build them. The best way -to do this is to ensure that the two interpreters are the same version of Python -(possibly different builds, or possibly copies of the same build). (Of course, -if your :option:`!--prefix` and :option:`!--exec-prefix` don't even point to an -alternate Python installation, this is immaterial.) - - -.. _inst-alt-install-prefix-windows: - -Alternate installation: Windows (the prefix scheme) ---------------------------------------------------- - -Windows has no concept of a user's home directory, and since the standard Python -installation under Windows is simpler than under Unix, the :option:`!--prefix` -option has traditionally been used to install additional packages in separate -locations on Windows. :: - - python setup.py install --prefix="\Temp\Python" - -to install modules to the :file:`\\Temp\\Python` directory on the current drive. - -The installation base is defined by the :option:`!--prefix` option; the -:option:`!--exec-prefix` option is not supported under Windows, which means that -pure Python modules and extension modules are installed into the same location. -Files are installed as follows: - -=============== ========================================================== -Type of file Installation directory -=============== ========================================================== -modules :file:`{prefix}\\Lib\\site-packages` -scripts :file:`{prefix}\\Scripts` -data :file:`{prefix}` -C headers :file:`{prefix}\\Include\\{distname}` -=============== ========================================================== - - -.. _inst-custom-install: - -Custom Installation -=================== - -Sometimes, the alternate installation schemes described in section -:ref:`inst-alt-install` just don't do what you want. You might want to tweak just -one or two directories while keeping everything under the same base directory, -or you might want to completely redefine the installation scheme. In either -case, you're creating a *custom installation scheme*. - -To create a custom installation scheme, you start with one of the alternate -schemes and override some of the installation directories used for the various -types of files, using these options: - -====================== ======================= -Type of file Override option -====================== ======================= -Python modules ``--install-purelib`` -extension modules ``--install-platlib`` -all modules ``--install-lib`` -scripts ``--install-scripts`` -data ``--install-data`` -C headers ``--install-headers`` -====================== ======================= - -These override options can be relative, absolute, -or explicitly defined in terms of one of the installation base directories. -(There are two installation base directories, and they are normally the -same---they only differ when you use the Unix "prefix scheme" and supply -different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` -will override values computed or given for ``--install-purelib`` and -``--install-platlib``, and is recommended for schemes that don't make a -difference between Python and extension modules.) - -For example, say you're installing a module distribution to your home directory -under Unix---but you want scripts to go in :file:`~/scripts` rather than -:file:`~/bin`. As you might expect, you can override this directory with the -:option:`!--install-scripts` option; in this case, it makes most sense to supply -a relative path, which will be interpreted relative to the installation base -directory (your home directory, in this case):: - - python setup.py install --home=~ --install-scripts=scripts - -Another Unix example: suppose your Python installation was built and installed -with a prefix of :file:`/usr/local/python`, so under a standard installation -scripts will wind up in :file:`/usr/local/python/bin`. If you want them in -:file:`/usr/local/bin` instead, you would supply this absolute directory for the -:option:`!--install-scripts` option:: - - python setup.py install --install-scripts=/usr/local/bin - -(This performs an installation using the "prefix scheme", where the prefix is -whatever your Python interpreter was installed with--- :file:`/usr/local/python` -in this case.) - -If you maintain Python on Windows, you might want third-party modules to live in -a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` -itself. This is almost as easy as customizing the script installation -directory---you just have to remember that there are two types of modules -to worry about, Python and extension modules, which can conveniently be both -controlled by one option:: - - python setup.py install --install-lib=Site - -The specified installation directory is relative to :file:`{prefix}`. Of -course, you also have to ensure that this directory is in Python's module -search path, such as by putting a :file:`.pth` file in a site directory (see -:mod:`site`). See section :ref:`inst-search-path` to find out how to modify -Python's search path. - -If you want to define an entire installation scheme, you just have to supply all -of the installation directory options. The recommended way to do this is to -supply relative paths; for example, if you want to maintain all Python -module-related files under :file:`python` in your home directory, and you want a -separate directory for each platform that you use your home directory from, you -might define the following installation scheme:: - - python setup.py install --home=~ \ - --install-purelib=python/lib \ - --install-platlib=python/lib.$PLAT \ - --install-scripts=python/scripts - --install-data=python/data - -or, equivalently, :: - - python setup.py install --home=~/python \ - --install-purelib=lib \ - --install-platlib='lib.$PLAT' \ - --install-scripts=scripts - --install-data=data - -``$PLAT`` is not (necessarily) an environment variable---it will be expanded by -the Distutils as it parses your command line options, just as it does when -parsing your configuration file(s). - -Obviously, specifying the entire installation scheme every time you install a -new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`): - -.. code-block:: ini - - [install] - install-base=$HOME - install-purelib=python/lib - install-platlib=python/lib.$PLAT - install-scripts=python/scripts - install-data=python/data - -or, equivalently, - -.. code-block:: ini - - [install] - install-base=$HOME/python - install-purelib=lib - install-platlib=lib.$PLAT - install-scripts=scripts - install-data=data - -Note that these two are *not* equivalent if you supply a different installation -base directory when you run the setup script. For example, :: - - python setup.py install --install-base=/tmp - -would install pure modules to :file:`/tmp/python/lib` in the first case, and -to :file:`/tmp/lib` in the second case. (For the second case, you probably -want to supply an installation base of :file:`/tmp/python`.) - -You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample -configuration file input. These are Distutils configuration variables, which -bear a strong resemblance to environment variables. In fact, you can use -environment variables in config files on platforms that have such a notion but -the Distutils additionally define a few extra variables that may not be in your -environment, such as ``$PLAT``. (And of course, on systems that don't have -environment variables, such as Mac OS 9, the configuration variables supplied by -the Distutils are the only ones you can use.) See section :ref:`inst-config-files` -for details. - -.. note:: When a :ref:`virtual environment ` is activated, any options - that change the installation path will be ignored from all distutils configuration - files to prevent inadvertently installing projects outside of the virtual - environment. - -.. XXX need some Windows examples---when would custom installation schemes be - needed on those platforms? - - -.. XXX Move this to Doc/using - -.. _inst-search-path: - -Modifying Python's Search Path ------------------------------- - -When the Python interpreter executes an :keyword:`import` statement, it searches -for both Python code and extension modules along a search path. A default value -for the path is configured into the Python binary when the interpreter is built. -You can determine the path by importing the :mod:`sys` module and printing the -value of ``sys.path``. :: - - $ python - Python 2.2 (#11, Oct 3 2002, 13:31:27) - [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', - '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', - '/usr/local/lib/python2.3/site-packages'] - >>> - -The null string in ``sys.path`` represents the current working directory. - -The expected convention for locally installed packages is to put them in the -:file:`{...}/site-packages/` directory, but you may want to install Python -modules into some arbitrary directory. For example, your site may have a -convention of keeping all software related to the web server under :file:`/www`. -Add-on Python modules might then belong in :file:`/www/python`, and in order to -import them, this directory must be added to ``sys.path``. There are several -different ways to add the directory. - -The most convenient way is to add a path configuration file to a directory -that's already on Python's path, usually to the :file:`.../site-packages/` -directory. Path configuration files have an extension of :file:`.pth`, and each -line must contain a single path that will be appended to ``sys.path``. (Because -the new paths are appended to ``sys.path``, modules in the added directories -will not override standard modules. This means you can't use this mechanism for -installing fixed versions of standard modules.) - -Paths can be absolute or relative, in which case they're relative to the -directory containing the :file:`.pth` file. See the documentation of -the :mod:`site` module for more information. - -A slightly less convenient way is to edit the :file:`site.py` file in Python's -standard library, and modify ``sys.path``. :file:`site.py` is automatically -imported when the Python interpreter is executed, unless the :option:`-S` switch -is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it: - -.. code-block:: python - - import sys - sys.path.append('/www/python/') - -However, if you reinstall the same major version of Python (perhaps when -upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by -the stock version. You'd have to remember that it was modified and save a copy -before doing the installation. - -There are two environment variables that can modify ``sys.path``. -:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python -installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, -the search path will be set to ``['', '/www/python/lib/pythonX.Y/', -'/www/python/lib/pythonX.Y/plat-linux2', ...]``. - -The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be -added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is -set to ``/www/python:/opt/py``, the search path will begin with -``['/www/python', '/opt/py']``. (Note that directories must exist in order to -be added to ``sys.path``; the :mod:`site` module removes paths that don't -exist.) - -Finally, ``sys.path`` is just a regular Python list, so any Python application -can modify it by adding or removing entries. - - -.. _inst-config-files: - -Distutils Configuration Files -============================= - -As mentioned above, you can use Distutils configuration files to record personal -or site preferences for any Distutils options. That is, any option to any -command can be stored in one of two or three (depending on your platform) -configuration files, which will be consulted before the command-line is parsed. -This means that configuration files will override default values, and the -command-line will in turn override configuration files. Furthermore, if -multiple configuration files apply, values from "earlier" files are overridden -by "later" files. - - -.. _inst-config-filenames: - -Location and names of config files ----------------------------------- - -The names and locations of the configuration files vary slightly across -platforms. On Unix and macOS, the three configuration files (in the order -they are processed) are: - -+--------------+----------------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+==========================================================+=======+ -| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | -+--------------+----------------------------------------------------------+-------+ -| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | -+--------------+----------------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+----------------------------------------------------------+-------+ - -And on Windows, the configuration files are: - -+--------------+-------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+=================================================+=======+ -| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | -+--------------+-------------------------------------------------+-------+ -| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | -+--------------+-------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+-------------------------------------------------+-------+ - -On all platforms, the "personal" file can be temporarily disabled by -passing the ``--no-user-cfg`` option. - -Notes: - -(1) - Strictly speaking, the system-wide configuration file lives in the directory - where the Distutils are installed; under Python 1.6 and later on Unix, this is - as shown. For Python 1.5.2, the Distutils will normally be installed to - :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system - configuration file should be put there under Python 1.5.2. - -(2) - On Unix, if the :envvar:`HOME` environment variable is not defined, the user's - home directory will be determined with the :func:`getpwuid` function from the - standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` - function used by Distutils. - -(3) - I.e., in the current directory (usually the location of the setup script). - -(4) - (See also note (1).) Under Python 1.6 and later, Python's default "installation - prefix" is :file:`C:\\Python`, so the system configuration file is normally - :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the - default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not - part of the standard library---so the system configuration file would be - :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python - 1.5.2 installation under Windows. - -(5) - On Windows, if the :envvar:`HOME` environment variable is not defined, - :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will - be tried. This is done by the :func:`os.path.expanduser` function used - by Distutils. - - -.. _inst-config-syntax: - -Syntax of config files ----------------------- - -The Distutils configuration files all have the same syntax. The config files -are grouped into sections. There is one section for each Distutils command, -plus a ``global`` section for global options that affect every command. Each -section consists of one option per line, specified as ``option=value``. - -For example, the following is a complete config file that just forces all -commands to run quietly by default: - -.. code-block:: ini - - [global] - verbose=0 - -If this is installed as the system config file, it will affect all processing of -any Python module distribution by any user on the current system. If it is -installed as your personal config file (on systems that support them), it will -affect only module distributions processed by you. And if it is used as the -:file:`setup.cfg` for a particular module distribution, it affects only that -distribution. - -You could override the default "build base" directory and make the -:command:`build\*` commands always forcibly rebuild all files with the -following: - -.. code-block:: ini - - [build] - build-base=blib - force=1 - -which corresponds to the command-line arguments :: - - python setup.py build --build-base=blib --force - -except that including the :command:`build` command on the command-line means -that command will be run. Including a particular command in config files has no -such implication; it only means that if the command is run, the options in the -config file will apply. (Or if other commands that derive values from it are -run, they will use the values in the config file.) - -You can find out the complete list of options for any command using the -:option:`!--help` option, e.g.:: - - python setup.py build --help - -and you can find out the complete list of global options by using -:option:`!--help` without a command:: - - python setup.py --help - -See also the "Reference" section of the "Distributing Python Modules" manual. - - -.. _inst-building-ext: - -Building Extensions: Tips and Tricks -==================================== - -Whenever possible, the Distutils try to use the configuration information made -available by the Python interpreter used to run the :file:`setup.py` script. -For example, the same compiler and linker flags used to compile Python will also -be used for compiling extensions. Usually this will work well, but in -complicated situations this might be inappropriate. This section discusses how -to override the usual Distutils behaviour. - - -.. _inst-tweak-flags: - -Tweaking compiler/linker flags ------------------------------- - -Compiling a Python extension written in C or C++ will sometimes require -specifying custom flags for the compiler and linker in order to use a particular -library or produce a special kind of object code. This is especially true if the -extension hasn't been tested on your platform, or if you're trying to -cross-compile Python. - -In the most general case, the extension author might have foreseen that -compiling the extensions would be complicated, and provided a :file:`Setup` file -for you to edit. This will likely only be done if the module distribution -contains many separate extension modules, or if they often require elaborate -sets of compiler flags in order to work. - -A :file:`Setup` file, if present, is parsed in order to get a list of extensions -to build. Each line in a :file:`Setup` describes a single module. Lines have -the following structure:: - - module ... [sourcefile ...] [cpparg ...] [library ...] - - -Let's examine each of the fields in turn. - -* *module* is the name of the extension module to be built, and should be a - valid Python identifier. You can't just change this in order to rename a module - (edits to the source code would also be needed), so this should be left alone. - -* *sourcefile* is anything that's likely to be a source code file, at least - judging by the filename. Filenames ending in :file:`.c` are assumed to be - written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are - assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed - to be in Objective C. - -* *cpparg* is an argument for the C preprocessor, and is anything starting with - :option:`!-I`, :option:`!-D`, :option:`!-U` or :option:`!-C`. - -* *library* is anything ending in :file:`.a` or beginning with :option:`!-l` or - :option:`!-L`. - -If a particular platform requires a special library on your platform, you can -add it by editing the :file:`Setup` file and running ``python setup.py build``. -For example, if the module defined by the line :: - - foo foomodule.c - -must be linked with the math library :file:`libm.a` on your platform, simply add -:option:`!-lm` to the line:: - - foo foomodule.c -lm - -Arbitrary switches intended for the compiler or the linker can be supplied with -the :option:`!-Xcompiler` *arg* and :option:`!-Xlinker` *arg* options:: - - foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm - -The next option after :option:`!-Xcompiler` and :option:`!-Xlinker` will be -appended to the proper command line, so in the above example the compiler will -be passed the :option:`!-o32` option, and the linker will be passed -:option:`!-shared`. If a compiler option requires an argument, you'll have to -supply multiple :option:`!-Xcompiler` options; for example, to pass ``-x c++`` -the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. - -Compiler flags can also be supplied through setting the :envvar:`CFLAGS` -environment variable. If set, the contents of :envvar:`CFLAGS` will be added to -the compiler flags specified in the :file:`Setup` file. - - -.. _inst-non-ms-compilers: - -Using non-Microsoft compilers on Windows ----------------------------------------- - -.. sectionauthor:: Rene Liebscher - - - -Borland/CodeGear C++ -^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the necessary steps to use Distutils with the Borland -C++ compiler version 5.5. First you have to know that Borland's object file -format (OMF) is different from the format used by the Python version you can -download from the Python or ActiveState web site. (Python is built with -Microsoft Visual C++, which uses COFF as the object file format.) For this -reason you have to convert Python's library :file:`python25.lib` into the -Borland format. You can do this as follows: - -.. Should we mention that users have to create cfg-files for the compiler? -.. see also http://community.borland.com/article/0,1410,21205,00.html - -:: - - coff2omf python25.lib python25_bcpp.lib - -The :file:`coff2omf` program comes with the Borland compiler. The file -:file:`python25.lib` is in the :file:`Libs` directory of your Python -installation. If your extension uses other libraries (zlib, ...) you have to -convert them too. - -The converted files have to reside in the same directories as the normal -libraries. - -How does Distutils manage to use these libraries with their changed names? If -the extension needs a library (eg. :file:`foo`) Distutils checks first if it -finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then -uses this library. In the case it doesn't find such a special library it uses -the default name (:file:`foo.lib`.) [#]_ - -To let Distutils compile your extension with Borland C++ you now have to type:: - - python setup.py build --compiler=bcpp - -If you want to use the Borland C++ compiler as the default, you could specify -this in your personal or system-wide configuration file for Distutils (see -section :ref:`inst-config-files`.) - - -.. seealso:: - - `C++Builder Compiler `_ - Information about the free C++ compiler from Borland, including links to the - download pages. - - `Creating Python Extensions Using Borland's Free Compiler `_ - Document describing how to use Borland's free command-line C++ compiler to build - Python. - - -GNU C / Cygwin / MinGW -^^^^^^^^^^^^^^^^^^^^^^ - -This section describes the necessary steps to use Distutils with the GNU C/C++ -compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter -that was built with Cygwin, everything should work without any of these -following steps. - -Not all extensions can be built with MinGW or Cygwin, but many can. Extensions -most likely to not work are those that use C++ or depend on Microsoft Visual C -extensions. - -To let Distutils compile your extension with Cygwin you have to type:: - - python setup.py build --compiler=cygwin - -and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: - - python setup.py build --compiler=mingw32 - -If you want to use any of these options/compilers as default, you should -consider writing it in your personal or system-wide configuration file for -Distutils (see section :ref:`inst-config-files`.) - -Older Versions of Python and MinGW -"""""""""""""""""""""""""""""""""" -The following instructions only apply if you're using a version of Python -inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with -binutils-2.13.90-20030111-1). - -These compilers require some special libraries. This task is more complex than -for Borland's C++, because there is no program to convert the library. First -you have to create a list of symbols which the Python DLL exports. (You can find -a good program for this task at -https://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). - -.. I don't understand what the next line means. --amk -.. (inclusive the references on data structures.) - -:: - - pexports python25.dll >python25.def - -The location of an installed :file:`python25.dll` will depend on the -installation options and the version and language of Windows. In a "just for -me" installation, it will appear in the root of the installation directory. In -a shared installation, it will be located in the system directory. - -Then you can create from these information an import library for gcc. :: - - /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a - -The resulting library has to be placed in the same directory as -:file:`python25.lib`. (Should be the :file:`libs` directory under your Python -installation directory.) - -If your extension uses other libraries (zlib,...) you might have to convert -them too. The converted files have to reside in the same directories as the -normal libraries do. - - -.. seealso:: - - `Building Python modules on MS Windows platform with MinGW `_ - Information about building the required libraries for the MinGW environment. - - -.. rubric:: Footnotes - -.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries - of the same name. - -.. [#] Check https://www.sourceware.org/cygwin/ for more information - -.. [#] Then you have no POSIX emulation available, but you also don't need - :file:`cygwin1.dll`. diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 5aec5178d48f3d..a46c1caefe4d8a 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -19,7 +19,9 @@ solutions to the common pool. This guide covers the installation part of the process. For a guide to creating and sharing your own Python projects, refer to the -:ref:`distribution guide `. +`Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ .. note:: diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst deleted file mode 100644 index d85ad94e9b7fe4..00000000000000 --- a/Doc/library/2to3.rst +++ /dev/null @@ -1,489 +0,0 @@ -.. _2to3-reference: - -2to3 --- Automated Python 2 to 3 code translation -================================================= - -.. sectionauthor:: Benjamin Peterson - -2to3 is a Python program that reads Python 2.x source code and applies a series -of *fixers* to transform it into valid Python 3.x code. The standard library -contains a rich set of fixers that will handle almost all code. 2to3 supporting -library :mod:`lib2to3` is, however, a flexible and generic library, so it is -possible to write your own fixers for 2to3. - -.. deprecated-removed:: 3.11 3.13 - The ``lib2to3`` module was marked pending for deprecation in Python 3.9 - (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated - in Python 3.11 (raising :exc:`DeprecationWarning`). The ``2to3`` tool is - part of that. It will be removed in Python 3.13. - -.. _2to3-using: - -Using 2to3 ----------- - -2to3 will usually be installed with the Python interpreter as a script. It is -also located in the :file:`Tools/scripts` directory of the Python root. - -2to3's basic arguments are a list of files or directories to transform. The -directories are recursively traversed for Python sources. - -Here is a sample Python 2.x source file, :file:`example.py`:: - - def greet(name): - print "Hello, {0}!".format(name) - print "What's your name?" - name = raw_input() - greet(name) - -It can be converted to Python 3.x code via 2to3 on the command line: - -.. code-block:: shell-session - - $ 2to3 example.py - -A diff against the original source file is printed. 2to3 can also write the -needed modifications right back to the source file. (A backup of the original -file is made unless :option:`!-n` is also given.) Writing the changes back is -enabled with the :option:`!-w` flag: - -.. code-block:: shell-session - - $ 2to3 -w example.py - -After transformation, :file:`example.py` looks like this:: - - def greet(name): - print("Hello, {0}!".format(name)) - print("What's your name?") - name = input() - greet(name) - -Comments and exact indentation are preserved throughout the translation process. - -By default, 2to3 runs a set of :ref:`predefined fixers <2to3-fixers>`. The -:option:`!-l` flag lists all available fixers. An explicit set of fixers to run -can be given with :option:`!-f`. Likewise the :option:`!-x` explicitly disables a -fixer. The following example runs only the ``imports`` and ``has_key`` fixers: - -.. code-block:: shell-session - - $ 2to3 -f imports -f has_key example.py - -This command runs every fixer except the ``apply`` fixer: - -.. code-block:: shell-session - - $ 2to3 -x apply example.py - -Some fixers are *explicit*, meaning they aren't run by default and must be -listed on the command line to be run. Here, in addition to the default fixers, -the ``idioms`` fixer is run: - -.. code-block:: shell-session - - $ 2to3 -f all -f idioms example.py - -Notice how passing ``all`` enables all default fixers. - -Sometimes 2to3 will find a place in your source code that needs to be changed, -but 2to3 cannot fix automatically. In this case, 2to3 will print a warning -beneath the diff for a file. You should address the warning in order to have -compliant 3.x code. - -2to3 can also refactor doctests. To enable this mode, use the :option:`!-d` -flag. Note that *only* doctests will be refactored. This also doesn't require -the module to be valid Python. For example, doctest like examples in a reST -document could also be refactored with this option. - -The :option:`!-v` option enables output of more information on the translation -process. - -Since some print statements can be parsed as function calls or statements, 2to3 -cannot always read files containing the print function. When 2to3 detects the -presence of the ``from __future__ import print_function`` compiler directive, it -modifies its internal grammar to interpret :func:`print` as a function. This -change can also be enabled manually with the :option:`!-p` flag. Use -:option:`!-p` to run fixers on code that already has had its print statements -converted. Also :option:`!-e` can be used to make :func:`exec` a function. - -The :option:`!-o` or :option:`!--output-dir` option allows specification of an -alternate directory for processed output files to be written to. The -:option:`!-n` flag is required when using this as backup files do not make sense -when not overwriting the input files. - -.. versionadded:: 3.2.3 - The :option:`!-o` option was added. - -The :option:`!-W` or :option:`!--write-unchanged-files` flag tells 2to3 to always -write output files even if no changes were required to the file. This is most -useful with :option:`!-o` so that an entire Python source tree is copied with -translation from one directory to another. -This option implies the :option:`!-w` flag as it would not make sense otherwise. - -.. versionadded:: 3.2.3 - The :option:`!-W` flag was added. - -The :option:`!--add-suffix` option specifies a string to append to all output -filenames. The :option:`!-n` flag is required when specifying this as backups -are not necessary when writing to different filenames. Example: - -.. code-block:: shell-session - - $ 2to3 -n -W --add-suffix=3 example.py - -Will cause a converted file named ``example.py3`` to be written. - -.. versionadded:: 3.2.3 - The :option:`!--add-suffix` option was added. - -To translate an entire project from one directory tree to another use: - -.. code-block:: shell-session - - $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode - - -.. _2to3-fixers: - -Fixers ------- - -Each step of transforming code is encapsulated in a fixer. The command ``2to3 --l`` lists them. As :ref:`documented above <2to3-using>`, each can be turned on -and off individually. They are described here in more detail. - - -.. 2to3fixer:: apply - - Removes usage of :func:`apply`. For example ``apply(function, *args, - **kwargs)`` is converted to ``function(*args, **kwargs)``. - -.. 2to3fixer:: asserts - - Replaces deprecated :mod:`unittest` method names with the correct ones. - - ================================ ========================================== - From To - ================================ ========================================== - ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b) - ` - ``assertEquals(a, b)`` :meth:`assertEqual(a, b) - ` - ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``failUnless(a)`` :meth:`assertTrue(a) - ` - ``assert_(a)`` :meth:`assertTrue(a) - ` - ``failIf(a)`` :meth:`assertFalse(a) - ` - ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal) - ` - ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ================================ ========================================== - -.. 2to3fixer:: basestring - - Converts :class:`basestring` to :class:`str`. - -.. 2to3fixer:: buffer - - Converts :class:`buffer` to :class:`memoryview`. This fixer is optional - because the :class:`memoryview` API is similar but not exactly the same as - that of :class:`buffer`. - -.. 2to3fixer:: dict - - Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to - :meth:`dict.items`, :meth:`dict.iterkeys` to :meth:`dict.keys`, and - :meth:`dict.itervalues` to :meth:`dict.values`. Similarly, - :meth:`dict.viewitems`, :meth:`dict.viewkeys` and :meth:`dict.viewvalues` are - converted respectively to :meth:`dict.items`, :meth:`dict.keys` and - :meth:`dict.values`. It also wraps existing usages of :meth:`dict.items`, - :meth:`dict.keys`, and :meth:`dict.values` in a call to :class:`list`. - -.. 2to3fixer:: except - - Converts ``except X, T`` to ``except X as T``. - -.. 2to3fixer:: exec - - Converts the ``exec`` statement to the :func:`exec` function. - -.. 2to3fixer:: execfile - - Removes usage of :func:`execfile`. The argument to :func:`execfile` is - wrapped in calls to :func:`open`, :func:`compile`, and :func:`exec`. - -.. 2to3fixer:: exitfunc - - Changes assignment of :attr:`sys.exitfunc` to use of the :mod:`atexit` - module. - -.. 2to3fixer:: filter - - Wraps :func:`filter` usage in a :class:`list` call. - -.. 2to3fixer:: funcattrs - - Fixes function attributes that have been renamed. For example, - ``my_function.func_closure`` is converted to ``my_function.__closure__``. - -.. 2to3fixer:: future - - Removes ``from __future__ import new_feature`` statements. - -.. 2to3fixer:: getcwdu - - Renames :func:`os.getcwdu` to :func:`os.getcwd`. - -.. 2to3fixer:: has_key - - Changes ``dict.has_key(key)`` to ``key in dict``. - -.. 2to3fixer:: idioms - - This optional fixer performs several transformations that make Python code - more idiomatic. Type comparisons like ``type(x) is SomeClass`` and - ``type(x) == SomeClass`` are converted to ``isinstance(x, SomeClass)``. - ``while 1`` becomes ``while True``. This fixer also tries to make use of - :func:`sorted` in appropriate places. For example, this block :: - - L = list(some_iterable) - L.sort() - - is changed to :: - - L = sorted(some_iterable) - -.. 2to3fixer:: import - - Detects sibling imports and converts them to relative imports. - -.. 2to3fixer:: imports - - Handles module renames in the standard library. - -.. 2to3fixer:: imports2 - - Handles other modules renames in the standard library. It is separate from - the :2to3fixer:`imports` fixer only because of technical limitations. - -.. 2to3fixer:: input - - Converts ``input(prompt)`` to ``eval(input(prompt))``. - -.. 2to3fixer:: intern - - Converts :func:`intern` to :func:`sys.intern`. - -.. 2to3fixer:: isinstance - - Fixes duplicate types in the second argument of :func:`isinstance`. For - example, ``isinstance(x, (int, int))`` is converted to ``isinstance(x, - int)`` and ``isinstance(x, (int, float, int))`` is converted to - ``isinstance(x, (int, float))``. - -.. 2to3fixer:: itertools_imports - - Removes imports of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap`. Imports of :func:`itertools.ifilterfalse` are also - changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: itertools - - Changes usage of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap` to their built-in equivalents. - :func:`itertools.ifilterfalse` is changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: long - - Renames :class:`long` to :class:`int`. - -.. 2to3fixer:: map - - Wraps :func:`map` in a :class:`list` call. It also changes ``map(None, x)`` - to ``list(x)``. Using ``from future_builtins import map`` disables this - fixer. - -.. 2to3fixer:: metaclass - - Converts the old metaclass syntax (``__metaclass__ = Meta`` in the class - body) to the new (``class X(metaclass=Meta)``). - -.. 2to3fixer:: methodattrs - - Fixes old method attribute names. For example, ``meth.im_func`` is converted - to ``meth.__func__``. - -.. 2to3fixer:: ne - - Converts the old not-equal syntax, ``<>``, to ``!=``. - -.. 2to3fixer:: next - - Converts the use of iterator's :meth:`~iterator.next` methods to the - :func:`next` function. It also renames :meth:`next` methods to - :meth:`~iterator.__next__`. - -.. 2to3fixer:: nonzero - - Renames definitions of methods called :meth:`__nonzero__` - to :meth:`~object.__bool__`. - -.. 2to3fixer:: numliterals - - Converts octal literals into the new syntax. - -.. 2to3fixer:: operator - - Converts calls to various functions in the :mod:`operator` module to other, - but equivalent, function calls. When needed, the appropriate ``import`` - statements are added, e.g. ``import collections.abc``. The following mapping - are made: - - ================================== ============================================= - From To - ================================== ============================================= - ``operator.isCallable(obj)`` ``callable(obj)`` - ``operator.sequenceIncludes(obj)`` ``operator.contains(obj)`` - ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.abc.Sequence)`` - ``operator.isMappingType(obj)`` ``isinstance(obj, collections.abc.Mapping)`` - ``operator.isNumberType(obj)`` ``isinstance(obj, numbers.Number)`` - ``operator.repeat(obj, n)`` ``operator.mul(obj, n)`` - ``operator.irepeat(obj, n)`` ``operator.imul(obj, n)`` - ================================== ============================================= - -.. 2to3fixer:: paren - - Add extra parenthesis where they are required in list comprehensions. For - example, ``[x for x in 1, 2]`` becomes ``[x for x in (1, 2)]``. - -.. 2to3fixer:: print - - Converts the ``print`` statement to the :func:`print` function. - -.. 2to3fixer:: raise - - Converts ``raise E, V`` to ``raise E(V)``, and ``raise E, V, T`` to ``raise - E(V).with_traceback(T)``. If ``E`` is a tuple, the translation will be - incorrect because substituting tuples for exceptions has been removed in 3.0. - -.. 2to3fixer:: raw_input - - Converts :func:`raw_input` to :func:`input`. - -.. 2to3fixer:: reduce - - Handles the move of :func:`reduce` to :func:`functools.reduce`. - -.. 2to3fixer:: reload - - Converts :func:`reload` to :func:`importlib.reload`. - -.. 2to3fixer:: renames - - Changes :data:`sys.maxint` to :data:`sys.maxsize`. - -.. 2to3fixer:: repr - - Replaces backtick repr with the :func:`repr` function. - -.. 2to3fixer:: set_literal - - Replaces use of the :class:`set` constructor with set literals. This fixer - is optional. - -.. 2to3fixer:: standarderror - - Renames :exc:`StandardError` to :exc:`Exception`. - -.. 2to3fixer:: sys_exc - - Changes the deprecated :data:`sys.exc_value`, :data:`sys.exc_type`, - :data:`sys.exc_traceback` to use :func:`sys.exc_info`. - -.. 2to3fixer:: throw - - Fixes the API change in generator's :meth:`throw` method. - -.. 2to3fixer:: tuple_params - - Removes implicit tuple parameter unpacking. This fixer inserts temporary - variables. - -.. 2to3fixer:: types - - Fixes code broken from the removal of some members in the :mod:`types` - module. - -.. 2to3fixer:: unicode - - Renames :class:`unicode` to :class:`str`. - -.. 2to3fixer:: urllib - - Handles the rename of :mod:`urllib` and :mod:`urllib2` to the :mod:`urllib` - package. - -.. 2to3fixer:: ws_comma - - Removes excess whitespace from comma separated items. This fixer is - optional. - -.. 2to3fixer:: xrange - - Renames :func:`xrange` to :func:`range` and wraps existing :func:`range` - calls with :class:`list`. - -.. 2to3fixer:: xreadlines - - Changes ``for x in file.xreadlines()`` to ``for x in file``. - -.. 2to3fixer:: zip - - Wraps :func:`zip` usage in a :class:`list` call. This is disabled when - ``from future_builtins import zip`` appears. - - -:mod:`lib2to3` --- 2to3's library ---------------------------------- - -.. module:: lib2to3 - :synopsis: The 2to3 library - -.. moduleauthor:: Guido van Rossum -.. moduleauthor:: Collin Winter -.. moduleauthor:: Benjamin Peterson - -**Source code:** :source:`Lib/lib2to3/` - --------------- - -.. deprecated-removed:: 3.11 3.13 - Python 3.9 switched to a PEG parser (see :pep:`617`) while lib2to3 is - using a less flexible LL(1) parser. Python 3.10 includes new language - syntax that is not parsable by lib2to3's LL(1) parser (see :pep:`634`). - The ``lib2to3`` module was marked pending for deprecation in Python 3.9 - (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated - in Python 3.11 (raising :exc:`DeprecationWarning`). - It will be removed from the standard library in Python 3.13. - Consider third-party alternatives such as `LibCST`_ or `parso`_. - -.. note:: - - The :mod:`lib2to3` API should be considered unstable and may change - drastically in the future. - -.. _LibCST: https://libcst.readthedocs.io/ -.. _parso: https://parso.readthedocs.io/ diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 8bd23daee73977..d261e4a4f338a5 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -22,42 +22,48 @@ can be inspected programmatically via importing :mod:`__future__` and examining its contents. -Each statement in :file:`__future__.py` is of the form:: +.. _future-classes: - FeatureName = _Feature(OptionalRelease, MandatoryRelease, - CompilerFlag) +.. class:: _Feature + Each statement in :file:`__future__.py` is of the form:: -where, normally, *OptionalRelease* is less than *MandatoryRelease*, and both are -5-tuples of the same form as :data:`sys.version_info`:: + FeatureName = _Feature(OptionalRelease, MandatoryRelease, + CompilerFlag) - (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int - PY_MINOR_VERSION, # the 1; an int - PY_MICRO_VERSION, # the 0; an int - PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string - PY_RELEASE_SERIAL # the 3; an int - ) + where, normally, *OptionalRelease* is less than *MandatoryRelease*, and both are + 5-tuples of the same form as :data:`sys.version_info`:: -*OptionalRelease* records the first release in which the feature was accepted. + (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int + PY_MINOR_VERSION, # the 1; an int + PY_MICRO_VERSION, # the 0; an int + PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string + PY_RELEASE_SERIAL # the 3; an int + ) -In the case of a *MandatoryRelease* that has not yet occurred, -*MandatoryRelease* predicts the release in which the feature will become part of -the language. +.. method:: _Feature.getOptionalRelease() -Else *MandatoryRelease* records when the feature became part of the language; in -releases at or after that, modules no longer need a future statement to use the -feature in question, but may continue to use such imports. + *OptionalRelease* records the first release in which the feature was accepted. -*MandatoryRelease* may also be ``None``, meaning that a planned feature got -dropped. +.. method:: _Feature.getMandatoryRelease() -Instances of class :class:`_Feature` have two corresponding methods, -:meth:`getOptionalRelease` and :meth:`getMandatoryRelease`. + In the case of a *MandatoryRelease* that has not yet occurred, + *MandatoryRelease* predicts the release in which the feature will become part of + the language. -*CompilerFlag* is the (bitfield) flag that should be passed in the fourth -argument to the built-in function :func:`compile` to enable the feature in -dynamically compiled code. This flag is stored in the :attr:`compiler_flag` -attribute on :class:`_Feature` instances. + Else *MandatoryRelease* records when the feature became part of the language; in + releases at or after that, modules no longer need a future statement to use the + feature in question, but may continue to use such imports. + + *MandatoryRelease* may also be ``None``, meaning that a planned feature got + dropped or that it is not yet decided. + +.. attribute:: _Feature.compiler_flag + + *CompilerFlag* is the (bitfield) flag that should be passed in the fourth + argument to the built-in function :func:`compile` to enable the feature in + dynamically compiled code. This flag is stored in the :attr:`_Feature.compiler_flag` + attribute on :class:`_Feature` instances. No feature description will ever be deleted from :mod:`__future__`. Since its introduction in Python 2.1 the following features have found their way into the diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index d29cbdff7830c8..c999253f781b10 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -54,45 +54,45 @@ The top-level code environment can be: * the scope of an interactive prompt:: - >>> __name__ - '__main__' + >>> __name__ + '__main__' * the Python module passed to the Python interpreter as a file argument: - .. code-block:: shell-session + .. code-block:: shell-session - $ python helloworld.py - Hello, world! + $ python helloworld.py + Hello, world! * the Python module or package passed to the Python interpreter with the :option:`-m` argument: - .. code-block:: shell-session + .. code-block:: shell-session - $ python -m tarfile - usage: tarfile.py [-h] [-v] (...) + $ python -m tarfile + usage: tarfile.py [-h] [-v] (...) * Python code read by the Python interpreter from standard input: - .. code-block:: shell-session + .. code-block:: shell-session - $ echo "import this" | python - The Zen of Python, by Tim Peters + $ echo "import this" | python + The Zen of Python, by Tim Peters - Beautiful is better than ugly. - Explicit is better than implicit. - ... + Beautiful is better than ugly. + Explicit is better than implicit. + ... * Python code passed to the Python interpreter with the :option:`-c` argument: - .. code-block:: shell-session + .. code-block:: shell-session - $ python -c "import this" - The Zen of Python, by Tim Peters + $ python -c "import this" + The Zen of Python, by Tim Peters - Beautiful is better than ugly. - Explicit is better than implicit. - ... + Beautiful is better than ugly. + Explicit is better than implicit. + ... In each of these situations, the top-level module's ``__name__`` is set to ``'__main__'``. @@ -102,9 +102,9 @@ top-level environment by checking its own ``__name__``, which allows a common idiom for conditionally executing code when the module is not initialized from an import statement:: - if __name__ == '__main__': - # Execute when the module is not initialized from an import statement. - ... + if __name__ == '__main__': + # Execute when the module is not initialized from an import statement. + ... .. seealso:: @@ -227,7 +227,7 @@ students:: import sys from .student import search_students - student_name = sys.argv[2] if len(sys.argv) >= 2 else '' + student_name = sys.argv[1] if len(sys.argv) >= 2 else '' print(f'Found student: {search_students(student_name)}') Note that ``from .student import search_students`` is an example of a relative @@ -238,9 +238,9 @@ package. For more details, see :ref:`intra-package-references` in the Idiomatic Usage ^^^^^^^^^^^^^^^ -The contents of ``__main__.py`` typically isn't fenced with -``if __name__ == '__main__'`` blocks. Instead, those files are kept short, -functions to execute from other modules. Those other modules can then be +The content of ``__main__.py`` typically isn't fenced with an +``if __name__ == '__main__'`` block. Instead, those files are kept +short and import functions to execute from other modules. Those other modules can then be easily unit-tested and are properly reusable. If used, an ``if __name__ == '__main__'`` block will still work as expected @@ -336,12 +336,12 @@ Note that importing ``__main__`` doesn't cause any issues with unintentionally running top-level code meant for script use which is put in the ``if __name__ == "__main__"`` block of the ``start`` module. Why does this work? -Python inserts an empty ``__main__`` module in :attr:`sys.modules` at +Python inserts an empty ``__main__`` module in :data:`sys.modules` at interpreter startup, and populates it by running top-level code. In our example this is the ``start`` module which runs line by line and imports ``namely``. In turn, ``namely`` imports ``__main__`` (which is really ``start``). That's an import cycle! Fortunately, since the partially populated ``__main__`` -module is present in :attr:`sys.modules`, Python passes that to ``namely``. +module is present in :data:`sys.modules`, Python passes that to ``namely``. See :ref:`Special considerations for __main__ ` in the import system's reference for details on how this works. diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index ba9314e46ab6ea..297f50a46e0692 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -70,10 +70,10 @@ This module defines the following constants and functions: there is no guarantee that the interruption will happen immediately. If given, *signum* is the number of the signal to simulate. - If *signum* is not given, :data:`signal.SIGINT` is simulated. + If *signum* is not given, :const:`signal.SIGINT` is simulated. If the given signal isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + :const:`signal.SIG_DFL` or :const:`signal.SIG_IGN`), this function does nothing. .. versionchanged:: 3.10 @@ -120,10 +120,13 @@ This module defines the following constants and functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. .. versionadded:: 3.8 + .. versionchanged:: 3.13 + Added support for GNU/kFreeBSD. + .. function:: stack_size([size]) @@ -150,8 +153,8 @@ This module defines the following constants and functions: .. data:: TIMEOUT_MAX The maximum value allowed for the *timeout* parameter of - :meth:`Lock.acquire`. Specifying a timeout greater than this value will - raise an :exc:`OverflowError`. + :meth:`Lock.acquire `. Specifying a timeout greater + than this value will raise an :exc:`OverflowError`. .. versionadded:: 3.2 @@ -208,7 +211,7 @@ In addition to these methods, lock objects can also be used via the **Caveats:** - .. index:: pair: module; signal +.. index:: pair: module; signal * Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` exception will be received by an arbitrary thread. (When the :mod:`signal` @@ -217,8 +220,9 @@ In addition to these methods, lock objects can also be used via the * Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is equivalent to calling :func:`_thread.exit`. -* It is not possible to interrupt the :meth:`acquire` method on a lock --- the - :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. +* It is not possible to interrupt the :meth:`~threading.Lock.acquire` method on + a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock + has been acquired. * When the main thread exits, it is system defined whether the other threads survive. On most systems, they are killed without executing diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 274b2d69f0ab5c..c073ea955abaa4 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -21,7 +21,7 @@ The :mod:`collections` module has some concrete classes that derive from ABCs; these can, of course, be further derived. In addition, the :mod:`collections.abc` submodule has some ABCs that can be used to test whether a class or instance provides a particular interface, for example, if it is -:term:`hashable` or if it is a mapping. +:term:`hashable` or if it is a :term:`mapping`. This module provides the metaclass :class:`ABCMeta` for defining ABCs and @@ -30,7 +30,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: .. class:: ABC A helper class that has :class:`ABCMeta` as its metaclass. With this class, - an abstract base class can be created by simply deriving from :class:`ABC` + an abstract base class can be created by simply deriving from :class:`!ABC` avoiding sometimes confusing metaclass usage, for example:: from abc import ABC @@ -38,11 +38,11 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: class MyABC(ABC): pass - Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore - inheriting from :class:`ABC` requires the usual precautions regarding + Note that the type of :class:`!ABC` is still :class:`ABCMeta`, therefore + inheriting from :class:`!ABC` requires the usual precautions regarding metaclass usage, as multiple inheritance may lead to metaclass conflicts. One may also define an abstract base class by passing the metaclass - keyword and using :class:`ABCMeta` directly, for example:: + keyword and using :class:`!ABCMeta` directly, for example:: from abc import ABCMeta @@ -65,7 +65,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: implementations defined by the registering ABC be callable (not even via :func:`super`). [#]_ - Classes created with a metaclass of :class:`ABCMeta` have the following method: + Classes created with a metaclass of :class:`!ABCMeta` have the following method: .. method:: register(subclass) @@ -86,7 +86,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: Returns the registered subclass, to allow usage as a class decorator. .. versionchanged:: 3.4 - To detect calls to :meth:`register`, you can use the + To detect calls to :meth:`!register`, you can use the :func:`get_cache_token` function. You can also override this method in an abstract base class: @@ -96,10 +96,10 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: (Must be defined as a class method.) Check whether *subclass* is considered a subclass of this ABC. This means - that you can customize the behavior of ``issubclass`` further without the + that you can customize the behavior of :func:`issubclass` further without the need to call :meth:`register` on every class you want to consider a subclass of the ABC. (This class method is called from the - :meth:`__subclasscheck__` method of the ABC.) + :meth:`~class.__subclasscheck__` method of the ABC.) This method should return ``True``, ``False`` or ``NotImplemented``. If it returns ``True``, the *subclass* is considered a subclass of this ABC. @@ -142,7 +142,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: The ABC ``MyIterable`` defines the standard iterable method, :meth:`~iterator.__iter__`, as an abstract method. The implementation given - here can still be called from subclasses. The :meth:`get_iterator` method + here can still be called from subclasses. The :meth:`!get_iterator` method is also part of the ``MyIterable`` abstract base class, but it does not have to be overridden in non-abstract derived classes. @@ -153,14 +153,14 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, even though it does not define an :meth:`~iterator.__iter__` method (it uses - the old-style iterable protocol, defined in terms of :meth:`__len__` and - :meth:`__getitem__`). Note that this will not make ``get_iterator`` + the old-style iterable protocol, defined in terms of :meth:`~object.__len__` and + :meth:`~object.__getitem__`). Note that this will not make ``get_iterator`` available as a method of ``Foo``, so it is provided separately. -The :mod:`abc` module also provides the following decorator: +The :mod:`!abc` module also provides the following decorator: .. decorator:: abstractmethod @@ -168,19 +168,19 @@ The :mod:`abc` module also provides the following decorator: Using this decorator requires that the class's metaclass is :class:`ABCMeta` or is derived from it. A class that has a metaclass derived from - :class:`ABCMeta` cannot be instantiated unless all of its abstract methods + :class:`!ABCMeta` cannot be instantiated unless all of its abstract methods and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`abstractmethod` may be used + of the normal 'super' call mechanisms. :func:`!abstractmethod` may be used to declare abstract methods for properties and descriptors. Dynamically adding abstract methods to a class, or attempting to modify the abstraction status of a method or class once it is created, are only supported using the :func:`update_abstractmethods` function. The - :func:`abstractmethod` only affects subclasses derived using regular - inheritance; "virtual subclasses" registered with the ABC's :meth:`register` - method are not affected. + :func:`!abstractmethod` only affects subclasses derived using regular + inheritance; "virtual subclasses" registered with the ABC's + :meth:`~ABCMeta.register` method are not affected. - When :func:`abstractmethod` is applied in combination with other method + When :func:`!abstractmethod` is applied in combination with other method descriptors, it should be applied as the innermost decorator, as shown in the following usage examples:: @@ -216,7 +216,7 @@ The :mod:`abc` module also provides the following decorator: In order to correctly interoperate with the abstract base class machinery, the descriptor must identify itself as abstract using - :attr:`__isabstractmethod__`. In general, this attribute should be ``True`` + :attr:`!__isabstractmethod__`. In general, this attribute should be ``True`` if any of the methods used to compose the descriptor are abstract. For example, Python's built-in :class:`property` does the equivalent of:: @@ -236,7 +236,7 @@ The :mod:`abc` module also provides the following decorator: super-call in a framework that uses cooperative multiple-inheritance. -The :mod:`abc` module also supports the following legacy decorators: +The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractclassmethod @@ -323,7 +323,7 @@ The :mod:`abc` module also supports the following legacy decorators: ... -The :mod:`abc` module also provides the following functions: +The :mod:`!abc` module also provides the following functions: .. function:: get_cache_token() diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst deleted file mode 100644 index 9f20a30193fa70..00000000000000 --- a/Doc/library/aifc.rst +++ /dev/null @@ -1,247 +0,0 @@ -:mod:`aifc` --- Read and write AIFF and AIFC files -================================================== - -.. module:: aifc - :synopsis: Read and write audio files in AIFF or AIFC format. - :deprecated: - -**Source code:** :source:`Lib/aifc.py` - -.. index:: - single: Audio Interchange File Format - single: AIFF - single: AIFF-C - - -.. deprecated-removed:: 3.11 3.13 - The :mod:`aifc` module is deprecated - (see :pep:`PEP 594 <594#aifc>` for details). - --------------- - -This module provides support for reading and writing AIFF and AIFF-C files. -AIFF is Audio Interchange File Format, a format for storing digital audio -samples in a file. AIFF-C is a newer version of the format that includes the -ability to compress the audio data. - -Audio files have a number of parameters that describe the audio data. The -sampling rate or frame rate is the number of times per second the sound is -sampled. The number of channels indicate if the audio is mono, stereo, or -quadro. Each frame consists of one sample per channel. The sample size is the -size in bytes of each sample. Thus a frame consists of -``nchannels * samplesize`` bytes, and a second's worth of audio consists of -``nchannels * samplesize * framerate`` bytes. - -For example, CD quality audio has a sample size of two bytes (16 bits), uses two -channels (stereo) and has a frame rate of 44,100 frames/second. This gives a -frame size of 4 bytes (2\*2), and a second's worth occupies 2\*2\*44100 bytes -(176,400 bytes). - -Module :mod:`aifc` defines the following function: - - -.. function:: open(file, mode=None) - - Open an AIFF or AIFF-C file and return an object instance with methods that are - described below. The argument *file* is either a string naming a file or a - :term:`file object`. *mode* must be ``'r'`` or ``'rb'`` when the file must be - opened for reading, or ``'w'`` or ``'wb'`` when the file must be opened for writing. - If omitted, ``file.mode`` is used if it exists, otherwise ``'rb'`` is used. When - used for writing, the file object should be seekable, unless you know ahead of - time how many samples you are going to write in total and use - :meth:`writeframesraw` and :meth:`setnframes`. - The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`!with` block completes, the :meth:`~aifc.close` method is called. - - .. versionchanged:: 3.4 - Support for the :keyword:`with` statement was added. - -Objects returned by :func:`.open` when a file is opened for reading have the -following methods: - - -.. method:: aifc.getnchannels() - - Return the number of audio channels (1 for mono, 2 for stereo). - - -.. method:: aifc.getsampwidth() - - Return the size in bytes of individual samples. - - -.. method:: aifc.getframerate() - - Return the sampling rate (number of audio frames per second). - - -.. method:: aifc.getnframes() - - Return the number of audio frames in the file. - - -.. method:: aifc.getcomptype() - - Return a bytes array of length 4 describing the type of compression - used in the audio file. For AIFF files, the returned value is - ``b'NONE'``. - - -.. method:: aifc.getcompname() - - Return a bytes array convertible to a human-readable description - of the type of compression used in the audio file. For AIFF files, - the returned value is ``b'not compressed'``. - - -.. method:: aifc.getparams() - - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. - - -.. method:: aifc.getmarkers() - - Return a list of markers in the audio file. A marker consists of a tuple of - three elements. The first is the mark ID (an integer), the second is the mark - position in frames from the beginning of the data (an integer), the third is the - name of the mark (a string). - - -.. method:: aifc.getmark(id) - - Return the tuple as described in :meth:`getmarkers` for the mark with the given - *id*. - - -.. method:: aifc.readframes(nframes) - - Read and return the next *nframes* frames from the audio file. The returned - data is a string containing for each frame the uncompressed samples of all - channels. - - -.. method:: aifc.rewind() - - Rewind the read pointer. The next :meth:`readframes` will start from the - beginning. - - -.. method:: aifc.setpos(pos) - - Seek to the specified frame number. - - -.. method:: aifc.tell() - - Return the current frame number. - - -.. method:: aifc.close() - - Close the AIFF file. After calling this method, the object can no longer be - used. - -Objects returned by :func:`.open` when a file is opened for writing have all the -above methods, except for :meth:`readframes` and :meth:`setpos`. In addition -the following methods exist. The :meth:`get\*` methods can only be called after -the corresponding :meth:`set\*` methods have been called. Before the first -:meth:`writeframes` or :meth:`writeframesraw`, all parameters except for the -number of frames must be filled in. - - -.. method:: aifc.aiff() - - Create an AIFF file. The default is that an AIFF-C file is created, unless the - name of the file ends in ``'.aiff'`` in which case the default is an AIFF file. - - -.. method:: aifc.aifc() - - Create an AIFF-C file. The default is that an AIFF-C file is created, unless - the name of the file ends in ``'.aiff'`` in which case the default is an AIFF - file. - - -.. method:: aifc.setnchannels(nchannels) - - Specify the number of channels in the audio file. - - -.. method:: aifc.setsampwidth(width) - - Specify the size in bytes of audio samples. - - -.. method:: aifc.setframerate(rate) - - Specify the sampling frequency in frames per second. - - -.. method:: aifc.setnframes(nframes) - - Specify the number of frames that are to be written to the audio file. If this - parameter is not set, or not set correctly, the file needs to support seeking. - - -.. method:: aifc.setcomptype(type, name) - - .. index:: - single: u-LAW - single: A-LAW - single: G.722 - - Specify the compression type. If not specified, the audio data will - not be compressed. In AIFF files, compression is not possible. - The name parameter should be a human-readable description of the - compression type as a bytes array, the type parameter should be a - bytes array of length 4. Currently the following compression types - are supported: ``b'NONE'``, ``b'ULAW'``, ``b'ALAW'``, ``b'G722'``. - - -.. method:: aifc.setparams(nchannels, sampwidth, framerate, comptype, compname) - - Set all the above parameters at once. The argument is a tuple consisting of the - various parameters. This means that it is possible to use the result of a - :meth:`getparams` call as argument to :meth:`setparams`. - - -.. method:: aifc.setmark(id, pos, name) - - Add a mark with the given id (larger than 0), and the given name at the given - position. This method can be called at any time before :meth:`close`. - - -.. method:: aifc.tell() - :noindex: - - Return the current write position in the output file. Useful in combination - with :meth:`setmark`. - - -.. method:: aifc.writeframes(data) - - Write data to the output file. This method can only be called after the audio - file parameters have been set. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: aifc.writeframesraw(data) - - Like :meth:`writeframes`, except that the header of the audio file is not - updated. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: aifc.close() - :noindex: - - Close the AIFF file. The header of the file is updated to reflect the actual - size of the audio data. After calling this method, the object can no longer be - used. - diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 33e367f3ccda89..fbffa71d200735 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -57,20 +57,20 @@ the extracted data in a :class:`argparse.Namespace` object:: Quick Links for add_argument() ------------------------------ -====================== =========================================================== ========================================================================================================================== -Name Description Values -====================== =========================================================== ========================================================================================================================== -action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'`` -choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance -const_ Store a constant value -default_ Default value used when an argument is not provided Defaults to ``None`` -dest_ Specify the attribute name used in the result namespace -help_ Help message for an argument -metavar_ Alternate display name for the argument as shown in help -nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'`` -required_ Indicate whether an argument is required or optional ``True`` or ``False`` -type_ Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function -====================== =========================================================== ========================================================================================================================== +============================ =========================================================== ========================================================================================================================== +Name Description Values +============================ =========================================================== ========================================================================================================================== +action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'`` +choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance +const_ Store a constant value +default_ Default value used when an argument is not provided Defaults to ``None`` +dest_ Specify the attribute name used in the result namespace +help_ Help message for an argument +metavar_ Alternate display name for the argument as shown in help +nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'`` +required_ Indicate whether an argument is required or optional ``True`` or ``False`` +:ref:`type ` Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function +============================ =========================================================== ========================================================================================================================== Example @@ -1132,7 +1132,7 @@ command-line argument was not present:: Namespace(foo='1') -.. _type: +.. _argparse-type: type ^^^^ diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 75c49e0f6d1ebe..ad622627724217 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -24,6 +24,8 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'u'`` | wchar_t | Unicode character | 2 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'w'`` | Py_UCS4 | Unicode character | 4 | | ++-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'H'`` | unsigned short | int | 2 | | @@ -51,11 +53,12 @@ Notes: It can be 16 bits or 32 bits depending on the platform. .. versionchanged:: 3.9 - ``array('u')`` now uses ``wchar_t`` as C type instead of deprecated + ``array('u')`` now uses :c:type:`wchar_t` as C type instead of deprecated ``Py_UNICODE``. This change doesn't affect its behavior because - ``Py_UNICODE`` is alias of ``wchar_t`` since Python 3.3. + ``Py_UNICODE`` is alias of :c:type:`wchar_t` since Python 3.3. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.16 + Please migrate to ``'w'`` typecode. The actual representation of values is determined by the machine architecture @@ -174,9 +177,9 @@ The module defines the following type: .. method:: fromunicode(s) - Extends this array with data from the given unicode string. The array must - be a type ``'u'`` array; otherwise a :exc:`ValueError` is raised. Use - ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an + Extends this array with data from the given unicode string. + The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. + Use ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an array of some other type. @@ -236,21 +239,22 @@ The module defines the following type: .. method:: tounicode() - Convert the array to a unicode string. The array must be a type ``'u'`` array; + Convert the array to a unicode string. The array must have a type ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to obtain a unicode string from an array of some other type. When an array object is printed or converted to a string, it is represented as ``array(typecode, initializer)``. The *initializer* is omitted if the array is -empty, otherwise it is a string if the *typecode* is ``'u'``, otherwise it is a -list of numbers. The string is guaranteed to be able to be converted back to an +empty, otherwise it is a string if the *typecode* is ``'u'`` or ``'w'``, +otherwise it is a list of numbers. +The string is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the :class:`~array.array` class has been imported using ``from array import array``. Examples:: array('l') - array('u', 'hello \u2641') + array('w', 'hello \u2641') array('l', [1, 2, 3, 4, 5]) array('d', [1.0, 2.0, 3.14]) @@ -260,10 +264,6 @@ Examples:: Module :mod:`struct` Packing and unpacking of heterogeneous binary data. - Module :mod:`xdrlib` - Packing and unpacking of External Data Representation (XDR) data as used in some - remote procedure call systems. - `NumPy `_ The NumPy package defines another array type. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 0811b3fa0e7842..4ebbe0e5471c88 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -146,6 +146,102 @@ Node classes Snakes `__ project and all its contributors. + +.. _ast-root-nodes: + +Root nodes +^^^^^^^^^^ + +.. class:: Module(body, type_ignores) + + A Python module, as with :ref:`file input `. + Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. + + *body* is a :class:`list` of the module's :ref:`ast-statements`. + + *type_ignores* is a :class:`list` of the module's type ignore comments; + see :func:`ast.parse` for more details. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x = 1'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Name(id='x', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + +.. class:: Expression(body) + + A single Python :ref:`expression input `. + Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. + + *body* is a single node, + one of the :ref:`expression types `. + + .. doctest:: + + >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) + Expression( + body=Constant(value=123)) + + +.. class:: Interactive(body) + + A single :ref:`interactive input `, like in :ref:`tut-interac`. + Node type generated by :func:`ast.parse` when *mode* is ``"single"``. + + *body* is a :class:`list` of :ref:`statement nodes `. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x = 1; y = 2', mode='single'), indent=4)) + Interactive( + body=[ + Assign( + targets=[ + Name(id='x', ctx=Store())], + value=Constant(value=1)), + Assign( + targets=[ + Name(id='y', ctx=Store())], + value=Constant(value=2))]) + + +.. class:: FunctionType(argtypes, returns) + + A representation of an old-style type comments for functions, + as Python versions prior to 3.5 didn't support :pep:`484` annotations. + Node type generated by :func:`ast.parse` when *mode* is ``"func_type"``. + + Such type comments would look like this:: + + def sum_two_number(a, b): + # type: (int, int) -> int + return a + b + + *argtypes* is a :class:`list` of :ref:`expression nodes `. + + *returns* is a single :ref:`expression node `. + + .. doctest:: + + >>> print(ast.dump(ast.parse('(int, str) -> List[int]', mode='func_type'), indent=4)) + FunctionType( + argtypes=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + returns=Subscript( + value=Name(id='List', ctx=Load()), + slice=Name(id='int', ctx=Load()), + ctx=Load())) + + .. versionadded:: 3.8 + + Literals ^^^^^^^^ @@ -344,6 +440,8 @@ Variables type_ignores=[]) +.. _ast-expressions: + Expressions ^^^^^^^^^^^ @@ -481,17 +579,17 @@ Expressions Comparison operator tokens. -.. class:: Call(func, args, keywords, starargs, kwargs) +.. class:: Call(func, args, keywords) A function call. ``func`` is the function, which will often be a :class:`Name` or :class:`Attribute` object. Of the arguments: * ``args`` holds a list of the arguments passed by position. - * ``keywords`` holds a list of :class:`keyword` objects representing + * ``keywords`` holds a list of :class:`.keyword` objects representing arguments passed by keyword. When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. ``starargs`` and ``kwargs`` are optional. + they can be empty lists. .. doctest:: @@ -552,10 +650,10 @@ Expressions .. class:: NamedExpr(target, value) - A named expression. This AST node is produced by the assignment expressions - operator (also known as the walrus operator). As opposed to the :class:`Assign` - node in which the first argument can be multiple nodes, in this case both - ``target`` and ``value`` must be single nodes. + A named expression. This AST node is produced by the assignment expressions + operator (also known as the walrus operator). As opposed to the :class:`Assign` + node in which the first argument can be multiple nodes, in this case both + ``target`` and ``value`` must be single nodes. .. doctest:: @@ -565,6 +663,7 @@ Expressions target=Name(id='x', ctx=Store()), value=Constant(value=4))) + .. versionadded:: 3.8 Subscripting ~~~~~~~~~~~~ @@ -735,6 +834,9 @@ Comprehensions ifs=[], is_async=1)])) + +.. _ast-statements: + Statements ^^^^^^^^^^ @@ -917,6 +1019,26 @@ Statements type_ignores=[]) +.. class:: TypeAlias(name, type_params, value) + + A :ref:`type alias ` created through the :keyword:`type` + statement. ``name`` is the name of the alias, ``type_params`` is a list of + :ref:`type parameters `, and ``value`` is the value of the + type alias. + + .. doctest:: + + >>> print(ast.dump(ast.parse('type Alias = int'), indent=4)) + Module( + body=[ + TypeAlias( + name=Name(id='Alias', ctx=Store()), + type_params=[], + value=Name(id='int', ctx=Load()))], + type_ignores=[]) + + .. versionadded:: 3.12 + Other statements which are only applicable inside functions or loops are described in other sections. @@ -1198,6 +1320,7 @@ Control flow finalbody=[])], type_ignores=[]) + .. versionadded:: 3.11 .. class:: ExceptHandler(type, name, body) @@ -1287,6 +1410,8 @@ Pattern matching that is being matched against the cases) and ``cases`` contains an iterable of :class:`match_case` nodes with the different cases. + .. versionadded:: 3.10 + .. class:: match_case(pattern, guard, body) A single case pattern in a ``match`` statement. ``pattern`` contains the @@ -1338,6 +1463,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchValue(value) A match literal or value pattern that compares by equality. ``value`` is @@ -1365,6 +1492,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchSingleton(value) A match literal pattern that compares by identity. ``value`` is the @@ -1390,6 +1519,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchSequence(patterns) A match sequence pattern. ``patterns`` contains the patterns to be matched @@ -1421,6 +1552,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchStar(name) Matches the rest of the sequence in a variable length match sequence pattern. @@ -1461,6 +1594,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchMapping(keys, patterns, rest) A match mapping pattern. ``keys`` is a sequence of expression nodes. @@ -1507,6 +1642,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchClass(cls, patterns, kwd_attrs, kwd_patterns) A match class pattern. ``cls`` is an expression giving the nominal class to @@ -1571,6 +1708,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchAs(pattern, name) A match "as-pattern", capture pattern or wildcard pattern. ``pattern`` @@ -1612,6 +1751,8 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + .. class:: MatchOr(patterns) A match "or-pattern". An or-pattern matches each of its subpatterns in turn @@ -1644,11 +1785,96 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) + .. versionadded:: 3.10 + +.. _ast-type-params: + +Type parameters +^^^^^^^^^^^^^^^ + +:ref:`Type parameters ` can exist on classes, functions, and type +aliases. + +.. class:: TypeVar(name, bound) + + A :class:`typing.TypeVar`. ``name`` is the name of the type variable. + ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, + it represents constraints; otherwise it represents the bound. + + .. doctest:: + + >>> print(ast.dump(ast.parse("type Alias[T: int] = list[T]"), indent=4)) + Module( + body=[ + TypeAlias( + name=Name(id='Alias', ctx=Store()), + type_params=[ + TypeVar( + name='T', + bound=Name(id='int', ctx=Load()))], + value=Subscript( + value=Name(id='list', ctx=Load()), + slice=Name(id='T', ctx=Load()), + ctx=Load()))], + type_ignores=[]) + + .. versionadded:: 3.12 + +.. class:: ParamSpec(name) + + A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + + .. doctest:: + + >>> print(ast.dump(ast.parse("type Alias[**P] = Callable[P, int]"), indent=4)) + Module( + body=[ + TypeAlias( + name=Name(id='Alias', ctx=Store()), + type_params=[ + ParamSpec(name='P')], + value=Subscript( + value=Name(id='Callable', ctx=Load()), + slice=Tuple( + elts=[ + Name(id='P', ctx=Load()), + Name(id='int', ctx=Load())], + ctx=Load()), + ctx=Load()))], + type_ignores=[]) + + .. versionadded:: 3.12 + +.. class:: TypeVarTuple(name) + + A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + + .. doctest:: + + >>> print(ast.dump(ast.parse("type Alias[*Ts] = tuple[*Ts]"), indent=4)) + Module( + body=[ + TypeAlias( + name=Name(id='Alias', ctx=Store()), + type_params=[ + TypeVarTuple(name='Ts')], + value=Subscript( + value=Name(id='tuple', ctx=Load()), + slice=Tuple( + elts=[ + Starred( + value=Name(id='Ts', ctx=Load()), + ctx=Load())], + ctx=Load()), + ctx=Load()))], + type_ignores=[]) + + .. versionadded:: 3.12 Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) +.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment, type_params) A function definition. @@ -1658,11 +1884,15 @@ Function and class definitions * ``decorator_list`` is the list of decorators to be applied, stored outermost first (i.e. the first in the list will be applied last). * ``returns`` is the return annotation. + * ``type_params`` is a list of :ref:`type parameters `. .. attribute:: type_comment ``type_comment`` is an optional string with the type annotation as a comment. + .. versionchanged:: 3.12 + Added ``type_params``. + .. class:: Lambda(args, body) @@ -1748,7 +1978,8 @@ Function and class definitions decorator_list=[ Name(id='decorator1', ctx=Load()), Name(id='decorator2', ctx=Load())], - returns=Constant(value='return annotation'))], + returns=Constant(value='return annotation'), + type_params=[])], type_ignores=[]) @@ -1819,21 +2050,19 @@ Function and class definitions type_ignores=[]) -.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list) +.. class:: ClassDef(name, bases, keywords, body, decorator_list, type_params) A class definition. * ``name`` is a raw string for the class name * ``bases`` is a list of nodes for explicitly specified base classes. - * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. + * ``keywords`` is a list of :class:`.keyword` nodes, principally for 'metaclass'. Other keywords will be passed to the metaclass, as per `PEP-3115 `_. - * ``starargs`` and ``kwargs`` are each a single node, as in a function call. - starargs will be expanded to join the list of base classes, and kwargs will - be passed to the metaclass. * ``body`` is a list of nodes representing the code within the class definition. * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. + * ``type_params`` is a list of :ref:`type parameters `. .. doctest:: @@ -1858,17 +2087,24 @@ Function and class definitions Pass()], decorator_list=[ Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())])], + Name(id='decorator2', ctx=Load())], + type_params=[])], type_ignores=[]) + .. versionchanged:: 3.12 + Added ``type_params``. + Async and await ^^^^^^^^^^^^^^^ -.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) +.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment, type_params) An ``async def`` function definition. Has the same fields as :class:`FunctionDef`. + .. versionchanged:: 3.12 + Added ``type_params``. + .. class:: Await(value) @@ -1898,7 +2134,8 @@ Async and await func=Name(id='other_func', ctx=Load()), args=[], keywords=[])))], - decorator_list=[])], + decorator_list=[], + type_params=[])], type_ignores=[]) @@ -1923,10 +2160,12 @@ Async and await Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, ast.PyCF_ONLY_AST)``. + filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` + and ``ast.PyCF_OPTIMIZED_AST`` otherwise. If ``type_comments=True`` is given, the parser is modified to check and return type comments as specified by :pep:`484` and :pep:`526`. @@ -1947,7 +2186,7 @@ and classes for traversing abstract syntax trees: Currently ``major`` must equal to ``3``. For example, setting ``feature_version=(3, 4)`` will allow the use of ``async`` and ``await`` as variable names. The lowest supported version is - ``(3, 4)``; the highest is ``sys.version_info[0:2]``. + ``(3, 7)``; the highest is ``sys.version_info[0:2]``. If source contains a null character ('\0'), :exc:`ValueError` is raised. @@ -1970,6 +2209,10 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.8 Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. + .. versionchanged:: 3.13 + The minimum supported version for feature_version is now (3,7) + The ``optimize`` argument was added. + .. function:: unparse(ast_obj) @@ -2240,26 +2483,26 @@ The following options are accepted: .. program:: ast -.. cmdoption:: -h, --help +.. option:: -h, --help Show the help message and exit. -.. cmdoption:: -m - --mode +.. option:: -m + --mode Specify what kind of code must be compiled, like the *mode* argument in :func:`parse`. -.. cmdoption:: --no-type-comments +.. option:: --no-type-comments Don't parse type comments. -.. cmdoption:: -a, --include-attributes +.. option:: -a, --include-attributes Include attributes such as line numbers and column offsets. -.. cmdoption:: -i - --indent +.. option:: -i + --indent Indentation of nodes in AST (number of spaces). diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 921a394a59fec7..a9c3a0183bb72d 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -34,7 +34,7 @@ There are several ways to enable asyncio debug mode: In addition to enabling the debug mode, consider also: * setting the log level of the :ref:`asyncio logger ` to - :py:data:`logging.DEBUG`, for example the following snippet of code + :py:const:`logging.DEBUG`, for example the following snippet of code can be run at startup of the application:: logging.basicConfig(level=logging.DEBUG) @@ -99,7 +99,7 @@ To schedule a coroutine object from a different OS thread, the # Wait for the result: result = future.result() -To handle signals and to execute subprocesses, the event loop must be +To handle signals the event loop must be run in the main thread. The :meth:`loop.run_in_executor` method can be used with a @@ -142,7 +142,7 @@ Logging asyncio uses the :mod:`logging` module and all logging is performed via the ``"asyncio"`` logger. -The default log level is :py:data:`logging.INFO`, which can be easily +The default log level is :py:const:`logging.INFO`, which can be easily adjusted:: logging.getLogger("asyncio").setLevel(logging.WARNING) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 8d0022cc66daac..ea1d146f06cf2b 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -243,9 +243,9 @@ Scheduling callbacks See the :ref:`concurrency and multithreading ` section of the documentation. -.. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. + .. versionchanged:: 3.7 + The *context* keyword-only parameter was added. See :pep:`567` + for more details. .. _asyncio-pass-keywords: @@ -403,11 +403,11 @@ Opening network connections Open a streaming transport connection to a given address specified by *host* and *port*. - The socket family can be either :py:data:`~socket.AF_INET` or - :py:data:`~socket.AF_INET6` depending on *host* (or the *family* + The socket family can be either :py:const:`~socket.AF_INET` or + :py:const:`~socket.AF_INET6` depending on *host* (or the *family* argument, if provided). - The socket type will be :py:data:`~socket.SOCK_STREAM`. + The socket type will be :py:const:`~socket.SOCK_STREAM`. *protocol_factory* must be a callable returning an :ref:`asyncio protocol ` implementation. @@ -509,7 +509,7 @@ Opening network connections .. versionchanged:: 3.6 - The socket option :py:data:`~socket.TCP_NODELAY` is set by default + The socket option :ref:`socket.TCP_NODELAY ` is set by default for all TCP connections. .. versionchanged:: 3.7 @@ -552,11 +552,11 @@ Opening network connections Create a datagram connection. - The socket family can be either :py:data:`~socket.AF_INET`, - :py:data:`~socket.AF_INET6`, or :py:data:`~socket.AF_UNIX`, + The socket family can be either :py:const:`~socket.AF_INET`, + :py:const:`~socket.AF_INET6`, or :py:const:`~socket.AF_UNIX`, depending on *host* (or the *family* argument, if provided). - The socket type will be :py:data:`~socket.SOCK_DGRAM`. + The socket type will be :py:const:`~socket.SOCK_DGRAM`. *protocol_factory* must be a callable returning a :ref:`protocol ` implementation. @@ -581,7 +581,7 @@ Opening network connections * *reuse_port* tells the kernel to allow this endpoint to be bound to the same port as other existing endpoints are bound to, so long as they all set this flag when being created. This option is not supported on Windows - and some Unixes. If the :py:data:`~socket.SO_REUSEPORT` constant is not + and some Unixes. If the :ref:`socket.SO_REUSEPORT ` constant is not defined then this capability is unsupported. * *allow_broadcast* tells the kernel to allow this endpoint to send @@ -607,7 +607,8 @@ Opening network connections .. versionchanged:: 3.8.1 The *reuse_address* parameter is no longer supported, as using - :py:data:`~sockets.SO_REUSEADDR` poses a significant security concern for + :ref:`socket.SO_REUSEADDR ` + poses a significant security concern for UDP. Explicitly passing ``reuse_address=True`` will raise an exception. When multiple processes with differing UIDs assign sockets to an @@ -616,7 +617,8 @@ Opening network connections For supported platforms, *reuse_port* can be used as a replacement for similar functionality. With *reuse_port*, - :py:data:`~sockets.SO_REUSEPORT` is used instead, which specifically + :ref:`socket.SO_REUSEPORT ` + is used instead, which specifically prevents processes with differing UIDs from assigning sockets to the same socket address. @@ -634,8 +636,8 @@ Opening network connections Create a Unix connection. - The socket family will be :py:data:`~socket.AF_UNIX`; socket - type will be :py:data:`~socket.SOCK_STREAM`. + The socket family will be :py:const:`~socket.AF_UNIX`; socket + type will be :py:const:`~socket.SOCK_STREAM`. A tuple of ``(transport, protocol)`` is returned on success. @@ -661,6 +663,8 @@ Opening network connections Creating network servers ^^^^^^^^^^^^^^^^^^^^^^^^ +.. _loop_create_server: + .. coroutinemethod:: loop.create_server(protocol_factory, \ host=None, port=None, *, \ family=socket.AF_UNSPEC, \ @@ -671,7 +675,7 @@ Creating network servers ssl_shutdown_timeout=None, \ start_serving=True) - Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) listening + Create a TCP server (socket type :const:`~socket.SOCK_STREAM`) listening on *port* of the *host* address. Returns a :class:`Server` object. @@ -699,10 +703,10 @@ Creating network servers be selected (note that if *host* resolves to multiple network interfaces, a different random port will be selected for each interface). - * *family* can be set to either :data:`socket.AF_INET` or - :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. + * *family* can be set to either :const:`socket.AF_INET` or + :const:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set, the *family* will be determined from host name - (defaults to :data:`~socket.AF_UNSPEC`). + (defaults to :const:`~socket.AF_UNSPEC`). * *flags* is a bitmask for :meth:`getaddrinfo`. @@ -756,7 +760,7 @@ Creating network servers .. versionchanged:: 3.6 Added *ssl_handshake_timeout* and *start_serving* parameters. - The socket option :py:data:`~socket.TCP_NODELAY` is set by default + The socket option :ref:`socket.TCP_NODELAY ` is set by default for all TCP connections. .. versionchanged:: 3.11 @@ -774,16 +778,20 @@ Creating network servers *, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, \ - start_serving=True) + start_serving=True, cleanup_socket=True) Similar to :meth:`loop.create_server` but works with the - :py:data:`~socket.AF_UNIX` socket family. + :py:const:`~socket.AF_UNIX` socket family. *path* is the name of a Unix domain socket, and is required, unless a *sock* argument is provided. Abstract Unix sockets, :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths are supported. + If *cleanup_socket* is True then the Unix socket will automatically + be removed from the filesystem when the server is closed, unless the + socket has been replaced after the server has been created. + See the documentation of the :meth:`loop.create_server` method for information about arguments to this method. @@ -798,6 +806,10 @@ Creating network servers Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.13 + + Added the *cleanup_socket* parameter. + .. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \ sock, *, ssl=None, ssl_handshake_timeout=None, \ @@ -895,6 +907,9 @@ TLS Upgrade object only because the coder caches *protocol*-side data and sporadically exchanges extra TLS session packets with *transport*. + In some situations (e.g. when the passed transport is already closing) this + may return ``None``. + Parameters: * *transport* and *protocol* instances that methods like @@ -1188,6 +1203,8 @@ Working with pipes Unix signals ^^^^^^^^^^^^ +.. _loop_add_signal_handler: + .. method:: loop.add_signal_handler(signum, callback, *args) Set *callback* as the handler for the *signum* signal. @@ -1388,6 +1405,14 @@ Enabling debug mode The new :ref:`Python Development Mode ` can now also be used to enable the debug mode. +.. attribute:: loop.slow_callback_duration + + This attribute can be used to set the + minimum execution duration in seconds that is considered "slow". + When debug mode is enabled, "slow" callbacks are logged. + + Default value is 100 milliseconds. + .. seealso:: The :ref:`debug mode of asyncio `. @@ -1408,6 +1433,8 @@ async/await code consider using the high-level :ref:`Subprocess Support on Windows ` for details. +.. _loop_subprocess_exec: + .. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) @@ -1439,6 +1466,7 @@ async/await code consider using the high-level * *stdin* can be any of these: * a file-like object + * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe()` * the :const:`subprocess.PIPE` constant (default) which will create a new pipe and connect it, * the value ``None`` which will make the subprocess inherit the file @@ -1590,6 +1618,9 @@ Do not instantiate the :class:`Server` class directly. .. versionchanged:: 3.7 Server object is an asynchronous context manager since Python 3.7. + .. versionchanged:: 3.11 + This class was exposed publicly as ``asyncio.Server`` in Python 3.9.11, 3.10.3 and 3.11. + .. method:: close() Stop serving: close listening sockets and set the :attr:`sockets` @@ -1598,8 +1629,9 @@ Do not instantiate the :class:`Server` class directly. The sockets that represent existing incoming client connections are left open. - The server is closed asynchronously, use the :meth:`wait_closed` - coroutine to wait until the server is closed. + The server is closed asynchronously; use the :meth:`wait_closed` + coroutine to wait until the server is closed (and no more + connections are active). .. method:: get_loop() @@ -1657,7 +1689,8 @@ Do not instantiate the :class:`Server` class directly. .. coroutinemethod:: wait_closed() - Wait until the :meth:`close` method completes. + Wait until the :meth:`close` method completes and all active + connections have finished. .. attribute:: sockets @@ -1679,13 +1712,13 @@ Event Loop Implementations asyncio ships with two different event loop implementations: :class:`SelectorEventLoop` and :class:`ProactorEventLoop`. -By default asyncio is configured to use :class:`SelectorEventLoop` -on Unix and :class:`ProactorEventLoop` on Windows. +By default asyncio is configured to use :class:`EventLoop`. .. class:: SelectorEventLoop - An event loop based on the :mod:`selectors` module. + A subclass of :class:`AbstractEventLoop` based on the + :mod:`selectors` module. Uses the most efficient *selector* available for the given platform. It is also possible to manually configure the @@ -1707,7 +1740,7 @@ on Unix and :class:`ProactorEventLoop` on Windows. .. class:: ProactorEventLoop - An event loop for Windows that uses "I/O Completion Ports" (IOCP). + A subclass of :class:`AbstractEventLoop` for Windows that uses "I/O Completion Ports" (IOCP). .. availability:: Windows. @@ -1716,6 +1749,14 @@ on Unix and :class:`ProactorEventLoop` on Windows. `MSDN documentation on I/O Completion Ports `_. +.. class:: EventLoop + + An alias to the most efficient available subclass of :class:`AbstractEventLoop` for the given + platform. + + It is an alias to :class:`SelectorEventLoop` on Unix and :class:`ProactorEventLoop` on Windows. + + .. versionadded:: 3.13 .. class:: AbstractEventLoop @@ -1865,7 +1906,7 @@ Set signal handlers for SIGINT and SIGTERM (This ``signals`` example only works on Unix.) -Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM` +Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` using the :meth:`loop.add_signal_handler` method:: import asyncio diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst index 9250f01b8a0895..7ad9103ca3fdfc 100644 --- a/Doc/library/asyncio-exceptions.rst +++ b/Doc/library/asyncio-exceptions.rst @@ -31,7 +31,7 @@ Exceptions .. versionchanged:: 3.8 - :exc:`CancelledError` is now a subclass of :class:`BaseException`. + :exc:`CancelledError` is now a subclass of :class:`BaseException` rather than :class:`Exception`. .. exception:: InvalidStateError diff --git a/Doc/library/asyncio-extending.rst b/Doc/library/asyncio-extending.rst index 8ffd356f2d1cc3..e7b293f484f8de 100644 --- a/Doc/library/asyncio-extending.rst +++ b/Doc/library/asyncio-extending.rst @@ -69,7 +69,7 @@ Task lifetime support ===================== A third party task implementation should call the following functions to keep a task -visible by :func:`asyncio.get_tasks` and :func:`asyncio.current_task`: +visible by :func:`asyncio.all_tasks` and :func:`asyncio.current_task`: .. function:: _register_task(task) diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 70cec9b2f90248..893ae5518f757d 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -276,4 +276,4 @@ the Future has a result:: :func:`concurrent.futures.as_completed` functions. - :meth:`asyncio.Future.cancel` accepts an optional ``msg`` argument, - but :func:`concurrent.futures.cancel` does not. + but :meth:`concurrent.futures.Future.cancel` does not. diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index 9ce48a24444e66..67136ba69ec875 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -484,19 +484,19 @@ Protocol classes can implement the following **callback methods**: :widths: 50 50 :class: full-width-table - * - ``callback`` :meth:`pipe_data_received() - ` + * - ``callback`` :meth:`~SubprocessProtocol.pipe_data_received` - Called when the child process writes data into its *stdout* or *stderr* pipe. - * - ``callback`` :meth:`pipe_connection_lost() - ` + * - ``callback`` :meth:`~SubprocessProtocol.pipe_connection_lost` - Called when one of the pipes communicating with the child process is closed. * - ``callback`` :meth:`process_exited() ` - - Called when the child process has exited. + - Called when the child process has exited. It can be called before + :meth:`~SubprocessProtocol.pipe_data_received` and + :meth:`~SubprocessProtocol.pipe_connection_lost` methods. Event Loop Policies diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 50ad8a2ab70324..19ec726c1be060 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -37,7 +37,7 @@ All event loops on Windows do not support the following methods: * :meth:`loop.create_unix_connection` and :meth:`loop.create_unix_server` are not supported. - The :data:`socket.AF_UNIX` socket family is specific to Unix. + The :const:`socket.AF_UNIX` socket family is specific to Unix. * :meth:`loop.add_signal_handler` and :meth:`loop.remove_signal_handler` are not supported. diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 7bc906eaafc1f2..3f734f544afe21 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -708,6 +708,9 @@ factories passed to the :meth:`loop.subprocess_exec` and Called when the child process has exited. + It can be called before :meth:`~SubprocessProtocol.pipe_data_received` and + :meth:`~SubprocessProtocol.pipe_connection_lost` methods. + Examples ======== @@ -746,7 +749,7 @@ received data, and close the connection:: loop = asyncio.get_running_loop() server = await loop.create_server( - lambda: EchoServerProtocol(), + EchoServerProtocol, '127.0.0.1', 8888) async with server: @@ -850,7 +853,7 @@ method, sends back received data:: # One protocol instance will be created to serve all # client requests. transport, protocol = await loop.create_datagram_endpoint( - lambda: EchoServerProtocol(), + EchoServerProtocol, local_addr=('127.0.0.1', 9999)) try: @@ -1003,12 +1006,26 @@ The subprocess is created by the :meth:`loop.subprocess_exec` method:: def __init__(self, exit_future): self.exit_future = exit_future self.output = bytearray() + self.pipe_closed = False + self.exited = False + + def pipe_connection_lost(self, fd, exc): + self.pipe_closed = True + self.check_for_exit() def pipe_data_received(self, fd, data): self.output.extend(data) def process_exited(self): - self.exit_future.set_result(True) + self.exited = True + # process_exited() method can be called before + # pipe_connection_lost() method: wait until both methods are + # called. + self.check_for_exit() + + def check_for_exit(self): + if self.pipe_closed and self.exited: + self.exit_future.set_result(True) async def get_date(): # Get a reference to the event loop as we plan to use diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index b68b2570ef071e..ec170dfde9e9aa 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -42,6 +42,8 @@ Running an asyncio Program This function should be used as a main entry point for asyncio programs, and should ideally only be called once. It is recommended to use *loop_factory* to configure the event loop instead of policies. + Passing :class:`asyncio.EventLoop` allows running asyncio without the + policy system. The executor is given a timeout duration of 5 minutes to shutdown. If the executor hasn't finished within that duration, a warning is diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index bbac1c32b5695f..0736e783bbc8c8 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -157,8 +157,8 @@ and work with streams: .. versionchanged:: 3.10 Removed the *loop* parameter. - .. versionchanged:: 3.11 - Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.11 + Added the *ssl_shutdown_timeout* parameter. .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ @@ -204,6 +204,10 @@ StreamReader directly; use :func:`open_connection` and :func:`start_server` instead. + .. method:: feed_eof() + + Acknowledge the EOF. + .. coroutinemethod:: read(n=-1) Read up to *n* bytes from the stream. diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index b7c83aa04c09f1..bf35b1cb798aee 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -68,7 +68,7 @@ Creating Subprocesses The *limit* argument sets the buffer limit for :class:`StreamReader` wrappers for :attr:`Process.stdout` and :attr:`Process.stderr` - (if :attr:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). + (if :const:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). Return a :class:`~asyncio.subprocess.Process` instance. @@ -86,7 +86,7 @@ Creating Subprocesses The *limit* argument sets the buffer limit for :class:`StreamReader` wrappers for :attr:`Process.stdout` and :attr:`Process.stderr` - (if :attr:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). + (if :const:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). Return a :class:`~asyncio.subprocess.Process` instance. @@ -249,7 +249,7 @@ their completion. Stop the child process. - On POSIX systems this method sends :py:data:`signal.SIGTERM` to the + On POSIX systems this method sends :py:const:`signal.SIGTERM` to the child process. On Windows the Win32 API function :c:func:`TerminateProcess` is diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index fe8d028150403d..797065c8ccf894 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -426,6 +426,9 @@ Sleeping .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.13 + Raises :exc:`ValueError` if *delay* is :data:`~math.nan`. + Running Tasks Concurrently ========================== @@ -628,9 +631,9 @@ Shielding From Cancellation Timeouts ======== -.. coroutinefunction:: timeout(delay) +.. function:: timeout(delay) - An :ref:`asynchronous context manager ` + Return an :ref:`asynchronous context manager ` that can be used to limit the amount of time spent waiting on something. @@ -651,16 +654,16 @@ Timeouts If ``long_running_task`` takes more than 10 seconds to complete, the context manager will cancel the current task and handle the resulting :exc:`asyncio.CancelledError` internally, transforming it - into an :exc:`asyncio.TimeoutError` which can be caught and handled. + into a :exc:`TimeoutError` which can be caught and handled. .. note:: The :func:`asyncio.timeout` context manager is what transforms - the :exc:`asyncio.CancelledError` into an :exc:`asyncio.TimeoutError`, - which means the :exc:`asyncio.TimeoutError` can only be caught + the :exc:`asyncio.CancelledError` into a :exc:`TimeoutError`, + which means the :exc:`TimeoutError` can only be caught *outside* of the context manager. - Example of catching :exc:`asyncio.TimeoutError`:: + Example of catching :exc:`TimeoutError`:: async def main(): try: @@ -721,7 +724,7 @@ Timeouts .. versionadded:: 3.11 -.. coroutinefunction:: timeout_at(when) +.. function:: timeout_at(when) Similar to :func:`asyncio.timeout`, except *when* is the absolute time to stop waiting, or ``None``. @@ -764,9 +767,6 @@ Timeouts If the wait is cancelled, the future *aw* is also cancelled. - .. versionchanged:: 3.10 - Removed the *loop* parameter. - .. _asyncio_example_waitfor: Example:: @@ -797,6 +797,9 @@ Timeouts .. versionchanged:: 3.10 Removed the *loop* parameter. + .. versionchanged:: 3.11 + Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. + Waiting Primitives ================== diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index c6a046f534e9a1..5f33c6813e74c0 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -46,9 +46,9 @@ Additionally, there are **low-level** APIs for *library and framework developers* to: * create and manage :ref:`event loops `, which - provide asynchronous APIs for :meth:`networking `, - running :meth:`subprocesses `, - handling :meth:`OS signals `, etc; + provide asynchronous APIs for :ref:`networking `, + running :ref:`subprocesses `, + handling :ref:`OS signals `, etc; * implement efficient protocols using :ref:`transports `; @@ -56,6 +56,8 @@ Additionally, there are **low-level** APIs for * :ref:`bridge ` callback-based libraries and code with async/await syntax. +.. _asyncio-cli: + You can experiment with an ``asyncio`` concurrent context in the REPL: .. code-block:: pycon diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index a2bd85b31c9a8d..3dbef69580d9b3 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -48,6 +48,16 @@ a cleanup function is undefined. This function returns *func*, which makes it possible to use it as a decorator. + .. warning:: + Starting new threads or calling :func:`os.fork` from a registered + function can lead to race condition between the main Python + runtime thread freeing thread states while internal :mod:`threading` + routines or the new process try to use that state. This can lead to + crashes rather than clean shutdown. + + .. versionchanged:: 3.12 + Attempts to start a new thread or :func:`os.fork` a new process + in a registered function now leads to :exc:`RuntimeError`. .. function:: unregister(func) diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst deleted file mode 100644 index 1f96575d08f5b1..00000000000000 --- a/Doc/library/audioop.rst +++ /dev/null @@ -1,287 +0,0 @@ -:mod:`audioop` --- Manipulate raw audio data -============================================ - -.. module:: audioop - :synopsis: Manipulate raw audio data. - :deprecated: - -.. deprecated-removed:: 3.11 3.13 - The :mod:`audioop` module is deprecated - (see :pep:`PEP 594 <594#audioop>` for details). - --------------- - -The :mod:`audioop` module contains some useful operations on sound fragments. -It operates on sound fragments consisting of signed integer samples 8, 16, 24 -or 32 bits wide, stored in :term:`bytes-like objects `. All scalar items are -integers, unless specified otherwise. - -.. versionchanged:: 3.4 - Support for 24-bit samples was added. - All functions now accept any :term:`bytes-like object`. - String input now results in an immediate error. - -.. index:: - single: Intel/DVI ADPCM - single: ADPCM, Intel/DVI - single: a-LAW - single: u-LAW - -This module provides support for a-LAW, u-LAW and Intel/DVI ADPCM encodings. - -.. This para is mostly here to provide an excuse for the index entries... - -A few of the more complicated operations only take 16-bit samples, otherwise the -sample size (in bytes) is always a parameter of the operation. - -The module defines the following variables and functions: - - -.. exception:: error - - This exception is raised on all errors, such as unknown number of bytes per - sample, etc. - - -.. function:: add(fragment1, fragment2, width) - - Return a fragment which is the addition of the two samples passed as parameters. - *width* is the sample width in bytes, either ``1``, ``2``, ``3`` or ``4``. Both - fragments should have the same length. Samples are truncated in case of overflow. - - -.. function:: adpcm2lin(adpcmfragment, width, state) - - Decode an Intel/DVI ADPCM coded fragment to a linear fragment. See the - description of :func:`lin2adpcm` for details on ADPCM coding. Return a tuple - ``(sample, newstate)`` where the sample has the width specified in *width*. - - -.. function:: alaw2lin(fragment, width) - - Convert sound fragments in a-LAW encoding to linearly encoded sound fragments. - a-LAW encoding always uses 8 bits samples, so *width* refers only to the sample - width of the output fragment here. - - -.. function:: avg(fragment, width) - - Return the average over all samples in the fragment. - - -.. function:: avgpp(fragment, width) - - Return the average peak-peak value over all samples in the fragment. No - filtering is done, so the usefulness of this routine is questionable. - - -.. function:: bias(fragment, width, bias) - - Return a fragment that is the original fragment with a bias added to each - sample. Samples wrap around in case of overflow. - - -.. function:: byteswap(fragment, width) - - "Byteswap" all samples in a fragment and returns the modified fragment. - Converts big-endian samples to little-endian and vice versa. - - .. versionadded:: 3.4 - - -.. function:: cross(fragment, width) - - Return the number of zero crossings in the fragment passed as an argument. - - -.. function:: findfactor(fragment, reference) - - Return a factor *F* such that ``rms(add(fragment, mul(reference, -F)))`` is - minimal, i.e., return the factor with which you should multiply *reference* to - make it match as well as possible to *fragment*. The fragments should both - contain 2-byte samples. - - The time taken by this routine is proportional to ``len(fragment)``. - - -.. function:: findfit(fragment, reference) - - Try to match *reference* as well as possible to a portion of *fragment* (which - should be the longer fragment). This is (conceptually) done by taking slices - out of *fragment*, using :func:`findfactor` to compute the best match, and - minimizing the result. The fragments should both contain 2-byte samples. - Return a tuple ``(offset, factor)`` where *offset* is the (integer) offset into - *fragment* where the optimal match started and *factor* is the (floating-point) - factor as per :func:`findfactor`. - - -.. function:: findmax(fragment, length) - - Search *fragment* for a slice of length *length* samples (not bytes!) with - maximum energy, i.e., return *i* for which ``rms(fragment[i*2:(i+length)*2])`` - is maximal. The fragments should both contain 2-byte samples. - - The routine takes time proportional to ``len(fragment)``. - - -.. function:: getsample(fragment, width, index) - - Return the value of sample *index* from the fragment. - - -.. function:: lin2adpcm(fragment, width, state) - - Convert samples to 4 bit Intel/DVI ADPCM encoding. ADPCM coding is an adaptive - coding scheme, whereby each 4 bit number is the difference between one sample - and the next, divided by a (varying) step. The Intel/DVI ADPCM algorithm has - been selected for use by the IMA, so it may well become a standard. - - *state* is a tuple containing the state of the coder. The coder returns a tuple - ``(adpcmfrag, newstate)``, and the *newstate* should be passed to the next call - of :func:`lin2adpcm`. In the initial call, ``None`` can be passed as the state. - *adpcmfrag* is the ADPCM coded fragment packed 2 4-bit values per byte. - - -.. function:: lin2alaw(fragment, width) - - Convert samples in the audio fragment to a-LAW encoding and return this as a - bytes object. a-LAW is an audio encoding format whereby you get a dynamic - range of about 13 bits using only 8 bit samples. It is used by the Sun audio - hardware, among others. - - -.. function:: lin2lin(fragment, width, newwidth) - - Convert samples between 1-, 2-, 3- and 4-byte formats. - - .. note:: - - In some audio formats, such as .WAV files, 16, 24 and 32 bit samples are - signed, but 8 bit samples are unsigned. So when converting to 8 bit wide - samples for these formats, you need to also add 128 to the result:: - - new_frames = audioop.lin2lin(frames, old_width, 1) - new_frames = audioop.bias(new_frames, 1, 128) - - The same, in reverse, has to be applied when converting from 8 to 16, 24 - or 32 bit width samples. - - -.. function:: lin2ulaw(fragment, width) - - Convert samples in the audio fragment to u-LAW encoding and return this as a - bytes object. u-LAW is an audio encoding format whereby you get a dynamic - range of about 14 bits using only 8 bit samples. It is used by the Sun audio - hardware, among others. - - -.. function:: max(fragment, width) - - Return the maximum of the *absolute value* of all samples in a fragment. - - -.. function:: maxpp(fragment, width) - - Return the maximum peak-peak value in the sound fragment. - - -.. function:: minmax(fragment, width) - - Return a tuple consisting of the minimum and maximum values of all samples in - the sound fragment. - - -.. function:: mul(fragment, width, factor) - - Return a fragment that has all samples in the original fragment multiplied by - the floating-point value *factor*. Samples are truncated in case of overflow. - - -.. function:: ratecv(fragment, width, nchannels, inrate, outrate, state[, weightA[, weightB]]) - - Convert the frame rate of the input fragment. - - *state* is a tuple containing the state of the converter. The converter returns - a tuple ``(newfragment, newstate)``, and *newstate* should be passed to the next - call of :func:`ratecv`. The initial call should pass ``None`` as the state. - - The *weightA* and *weightB* arguments are parameters for a simple digital filter - and default to ``1`` and ``0`` respectively. - - -.. function:: reverse(fragment, width) - - Reverse the samples in a fragment and returns the modified fragment. - - -.. function:: rms(fragment, width) - - Return the root-mean-square of the fragment, i.e. ``sqrt(sum(S_i^2)/n)``. - - This is a measure of the power in an audio signal. - - -.. function:: tomono(fragment, width, lfactor, rfactor) - - Convert a stereo fragment to a mono fragment. The left channel is multiplied by - *lfactor* and the right channel by *rfactor* before adding the two channels to - give a mono signal. - - -.. function:: tostereo(fragment, width, lfactor, rfactor) - - Generate a stereo fragment from a mono fragment. Each pair of samples in the - stereo fragment are computed from the mono sample, whereby left channel samples - are multiplied by *lfactor* and right channel samples by *rfactor*. - - -.. function:: ulaw2lin(fragment, width) - - Convert sound fragments in u-LAW encoding to linearly encoded sound fragments. - u-LAW encoding always uses 8 bits samples, so *width* refers only to the sample - width of the output fragment here. - -Note that operations such as :func:`.mul` or :func:`.max` make no distinction -between mono and stereo fragments, i.e. all samples are treated equal. If this -is a problem the stereo fragment should be split into two mono fragments first -and recombined later. Here is an example of how to do that:: - - def mul_stereo(sample, width, lfactor, rfactor): - lsample = audioop.tomono(sample, width, 1, 0) - rsample = audioop.tomono(sample, width, 0, 1) - lsample = audioop.mul(lsample, width, lfactor) - rsample = audioop.mul(rsample, width, rfactor) - lsample = audioop.tostereo(lsample, width, 1, 0) - rsample = audioop.tostereo(rsample, width, 0, 1) - return audioop.add(lsample, rsample, width) - -If you use the ADPCM coder to build network packets and you want your protocol -to be stateless (i.e. to be able to tolerate packet loss) you should not only -transmit the data but also the state. Note that you should send the *initial* -state (the one you passed to :func:`lin2adpcm`) along to the decoder, not the -final state (as returned by the coder). If you want to use -:class:`struct.Struct` to store the state in binary you can code the first -element (the predicted value) in 16 bits and the second (the delta index) in 8. - -The ADPCM coders have never been tried against other ADPCM coders, only against -themselves. It could well be that I misinterpreted the standards in which case -they will not be interoperable with the respective standards. - -The :func:`find\*` routines might look a bit funny at first sight. They are -primarily meant to do echo cancellation. A reasonably fast way to do this is to -pick the most energetic piece of the output sample, locate that in the input -sample and subtract the whole output sample from the input sample:: - - def echocancel(outputdata, inputdata): - pos = audioop.findmax(outputdata, 800) # one tenth second - out_test = outputdata[pos*2:] - in_test = inputdata[pos*2:] - ipos, factor = audioop.findfit(in_test, out_test) - # Optional (for better cancellation): - # factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)], - # out_test) - prefill = '\0'*(pos+ipos)*2 - postfill = '\0'*(len(inputdata)-len(prefill)-len(outputdata)) - outputdata = prefill + audioop.mul(outputdata, 2, -factor) + postfill - return audioop.add(inputdata, outputdata, 2) - diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 21960cb7972e6e..39fabb59bb1984 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -6,14 +6,13 @@ representations. .. index:: - pair: module; uu pair: module; base64 -------------- The :mod:`binascii` module contains a number of methods to convert between binary and various ASCII-encoded binary representations. Normally, you will not -use these functions directly but use wrapper modules like :mod:`uu` or +use these functions directly but use wrapper modules like :mod:`base64` instead. The :mod:`binascii` module contains low-level functions written in C for greater speed that are used by the higher-level modules. @@ -58,10 +57,11 @@ The :mod:`binascii` module defines the following functions: data will raise :exc:`binascii.Error`. Valid base64: - * Conforms to :rfc:`3548`. - * Contains only characters from the base64 alphabet. - * Contains no excess data after padding (including excess padding, newlines, etc.). - * Does not start with a padding. + + * Conforms to :rfc:`3548`. + * Contains only characters from the base64 alphabet. + * Contains no excess data after padding (including excess padding, newlines, etc.). + * Does not start with a padding. .. versionchanged:: 3.11 Added the *strict_mode* parameter. @@ -179,8 +179,5 @@ The :mod:`binascii` module defines the following functions: Support for RFC compliant base64-style encoding in base 16, 32, 64, and 85. - Module :mod:`uu` - Support for UU encoding used on Unix. - Module :mod:`quopri` Support for quoted-printable encoding used in MIME email messages. diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 32df99869eb530..6a95a4a6e8d183 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -87,10 +87,11 @@ The :mod:`bz2` module contains: compressed streams. :class:`BZ2File` provides all of the members specified by the - :class:`io.BufferedIOBase`, except for :meth:`detach` and :meth:`truncate`. + :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach` + and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - :class:`BZ2File` also provides the following method: + :class:`BZ2File` also provides the following methods: .. method:: peek([n]) @@ -105,14 +106,52 @@ The :mod:`bz2` module contains: .. versionadded:: 3.3 + .. method:: fileno() + + Return the file descriptor for the underlying file. + + .. versionadded:: 3.3 + + .. method:: readable() + + Return whether the file was opened for reading. + + .. versionadded:: 3.3 + + .. method:: seekable() + + Return whether the file supports seeking. + + .. versionadded:: 3.3 + + .. method:: writable() + + Return whether the file was opened for writing. + + .. versionadded:: 3.3 + + .. method:: read1(size=-1) + + Read up to *size* uncompressed bytes, while trying to avoid + making multiple reads from the underlying stream. Reads up to a + buffer's worth of data if size is negative. + + Returns ``b''`` if the file is at EOF. + + .. versionadded:: 3.3 + + .. method:: readinto(b) + + Read bytes into *b*. + + Returns the number of bytes read (0 for EOF). + + .. versionadded:: 3.3 + .. versionchanged:: 3.1 Support for the :keyword:`with` statement was added. - .. versionchanged:: 3.3 - The :meth:`fileno`, :meth:`readable`, :meth:`seekable`, :meth:`writable`, - :meth:`read1` and :meth:`readinto` methods were added. - .. versionchanged:: 3.3 Support was added for *filename* being a :term:`file object` instead of an actual filename. diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 07d04a1c7b582a..157a7537f97dc6 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -28,58 +28,6 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is 2 BC, and so on. -.. class:: Day - - Enumeration defining the days of the week as integer constants, from 0 to 6. - - .. attribute:: MONDAY - - .. attribute:: TUESDAY - - .. attribute:: WEDNESDAY - - .. attribute:: THURSDAY - - .. attribute:: FRIDAY - - .. attribute:: SATURDAY - - .. attribute:: SUNDAY - - .. versionadded:: 3.12 - - -.. class:: Month - - Enumeration defining months of the year as integer constants, from 1 to 12. - - .. attribute:: JANUARY - - .. attribute:: FEBRUARY - - .. attribute:: MARCH - - .. attribute:: APRIL - - .. attribute:: MAY - - .. attribute:: JUNE - - .. attribute:: JULY - - .. attribute:: AUGUST - - .. attribute:: SEPTEMBER - - .. attribute:: OCTOBER - - .. attribute:: NOVEMBER - - .. attribute:: DECEMBER - - .. versionadded:: 3.12 - - .. class:: Calendar(firstweekday=0) Creates a :class:`Calendar` object. *firstweekday* is an integer specifying the @@ -446,6 +394,29 @@ The :mod:`calendar` module exports the following data attributes: An array that represents the abbreviated days of the week in the current locale. +.. data:: MONDAY + TUESDAY + WEDNESDAY + THURSDAY + FRIDAY + SATURDAY + SUNDAY + + Aliases for the days of the week, + where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``. + + .. versionadded:: 3.12 + + +.. class:: Day + + Enumeration defining days of the week as integer constants. + The members of this enumeration are exported to the module scope as + :data:`MONDAY` through :data:`SUNDAY`. + + .. versionadded:: 3.12 + + .. data:: month_name An array that represents the months of the year in the current locale. This @@ -459,15 +430,56 @@ The :mod:`calendar` module exports the following data attributes: locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_abbr[0]`` is the empty string. -.. data:: MONDAY - TUESDAY - WEDNESDAY - THURSDAY - FRIDAY - SATURDAY - SUNDAY - Aliases for day numbers, where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``. +.. data:: JANUARY + FEBRUARY + MARCH + APRIL + MAY + JUNE + JULY + AUGUST + SEPTEMBER + OCTOBER + NOVEMBER + DECEMBER + + Aliases for the months of the year, + where ``JANUARY`` is ``1`` and ``DECEMBER`` is ``12``. + + .. versionadded:: 3.12 + + +.. class:: Month + + Enumeration defining months of the year as integer constants. + The members of this enumeration are exported to the module scope as + :data:`JANUARY` through :data:`DECEMBER`. + + .. versionadded:: 3.12 + + +The :mod:`calendar` module defines the following exceptions: + +.. exception:: IllegalMonthError(month) + + A subclass of :exc:`ValueError`, + raised when the given month number is outside of the range 1-12 (inclusive). + + .. attribute:: month + + The invalid month number. + + +.. exception:: IllegalWeekdayError(weekday) + + A subclass of :exc:`ValueError`, + raised when the given weekday number is outside of the range 0-6 (inclusive). + + .. attribute:: weekday + + The invalid weekday number. + .. seealso:: @@ -477,3 +489,146 @@ The :mod:`calendar` module exports the following data attributes: Module :mod:`time` Low-level time related functions. + + +.. _calendar-cli: + +Command-Line Usage +------------------ + +.. versionadded:: 2.5 + +The :mod:`calendar` module can be executed as a script from the command line +to interactively print a calendar. + +.. code-block:: shell + + python -m calendar [-h] [-L LOCALE] [-e ENCODING] [-t {text,html}] + [-w WIDTH] [-l LINES] [-s SPACING] [-m MONTHS] [-c CSS] + [year] [month] + + +For example, to print a calendar for the year 2000: + +.. code-block:: console + + $ python -m calendar 2000 + 2000 + + January February March + Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 1 2 3 4 5 + 3 4 5 6 7 8 9 7 8 9 10 11 12 13 6 7 8 9 10 11 12 + 10 11 12 13 14 15 16 14 15 16 17 18 19 20 13 14 15 16 17 18 19 + 17 18 19 20 21 22 23 21 22 23 24 25 26 27 20 21 22 23 24 25 26 + 24 25 26 27 28 29 30 28 29 27 28 29 30 31 + 31 + + April May June + Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 7 1 2 3 4 + 3 4 5 6 7 8 9 8 9 10 11 12 13 14 5 6 7 8 9 10 11 + 10 11 12 13 14 15 16 15 16 17 18 19 20 21 12 13 14 15 16 17 18 + 17 18 19 20 21 22 23 22 23 24 25 26 27 28 19 20 21 22 23 24 25 + 24 25 26 27 28 29 30 29 30 31 26 27 28 29 30 + + July August September + Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 1 2 3 + 3 4 5 6 7 8 9 7 8 9 10 11 12 13 4 5 6 7 8 9 10 + 10 11 12 13 14 15 16 14 15 16 17 18 19 20 11 12 13 14 15 16 17 + 17 18 19 20 21 22 23 21 22 23 24 25 26 27 18 19 20 21 22 23 24 + 24 25 26 27 28 29 30 28 29 30 31 25 26 27 28 29 30 + 31 + + October November December + Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 1 2 3 4 5 1 2 3 + 2 3 4 5 6 7 8 6 7 8 9 10 11 12 4 5 6 7 8 9 10 + 9 10 11 12 13 14 15 13 14 15 16 17 18 19 11 12 13 14 15 16 17 + 16 17 18 19 20 21 22 20 21 22 23 24 25 26 18 19 20 21 22 23 24 + 23 24 25 26 27 28 29 27 28 29 30 25 26 27 28 29 30 31 + 30 31 + + +The following options are accepted: + +.. program:: calendar + + +.. option:: --help, -h + + Show the help message and exit. + + +.. option:: --locale LOCALE, -L LOCALE + + The locale to use for month and weekday names. + Defaults to English. + + +.. option:: --encoding ENCODING, -e ENCODING + + The encoding to use for output. + :option:`--encoding` is required if :option:`--locale` is set. + + +.. option:: --type {text,html}, -t {text,html} + + Print the calendar to the terminal as text, + or as an HTML document. + + +.. option:: year + + The year to print the calendar for. + Must be a number between 1 and 9999. + Defaults to the current year. + + +.. option:: month + + The month of the specified :option:`year` to print the calendar for. + Must be a number between 1 and 12, + and may only be used in text mode. + Defaults to printing a calendar for the full year. + + +*Text-mode options:* + +.. option:: --width WIDTH, -w WIDTH + + The width of the date column in terminal columns. + The date is printed centred in the column. + Any value lower than 2 is ignored. + Defaults to 2. + + +.. option:: --lines LINES, -l LINES + + The number of lines for each week in terminal rows. + The date is printed top-aligned. + Any value lower than 1 is ignored. + Defaults to 1. + + +.. option:: --spacing SPACING, -s SPACING + + The space between months in columns. + Any value lower than 2 is ignored. + Defaults to 6. + + +.. option:: --months MONTHS, -m MONTHS + + The number of months printed per row. + Defaults to 3. + + +*HTML-mode options:* + +.. option:: --css CSS, -c CSS + + The path of a CSS stylesheet to use for the calendar. + This must either be relative to the generated HTML, + or an absolute HTTP or ``file:///`` URL. diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst deleted file mode 100644 index 295a601a7bf197..00000000000000 --- a/Doc/library/cgi.rst +++ /dev/null @@ -1,564 +0,0 @@ -:mod:`cgi` --- Common Gateway Interface support -=============================================== - -.. module:: cgi - :synopsis: Helpers for running Python scripts via the Common Gateway Interface. - :deprecated: - -**Source code:** :source:`Lib/cgi.py` - -.. index:: - pair: WWW; server - pair: CGI; protocol - pair: HTTP; protocol - pair: MIME; headers - single: URL - single: Common Gateway Interface - -.. deprecated-removed:: 3.11 3.13 - The :mod:`cgi` module is deprecated - (see :pep:`PEP 594 <594#cgi>` for details and alternatives). - - The :class:`FieldStorage` class can typically be replaced with - :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, - and the :mod:`email.message` module or - `multipart `_ for ``POST`` and ``PUT``. - Most :ref:`utility functions ` have replacements. - --------------- - -Support module for Common Gateway Interface (CGI) scripts. - -This module defines a number of utilities for use by CGI scripts written in -Python. - -The global variable ``maxlen`` can be set to an integer indicating the maximum -size of a POST request. POST requests larger than this size will result in a -:exc:`ValueError` being raised during parsing. The default value of this -variable is ``0``, meaning the request size is unlimited. - -.. include:: ../includes/wasm-notavail.rst - -Introduction ------------- - -.. _cgi-intro: - -A CGI script is invoked by an HTTP server, usually to process user input -submitted through an HTML ``
`` or ```` element. - -Most often, CGI scripts live in the server's special :file:`cgi-bin` directory. -The HTTP server places all sorts of information about the request (such as the -client's hostname, the requested URL, the query string, and lots of other -goodies) in the script's shell environment, executes the script, and sends the -script's output back to the client. - -The script's input is connected to the client too, and sometimes the form data -is read this way; at other times the form data is passed via the "query string" -part of the URL. This module is intended to take care of the different cases -and provide a simpler interface to the Python script. It also provides a number -of utilities that help in debugging scripts, and the latest addition is support -for file uploads from a form (if your browser supports it). - -The output of a CGI script should consist of two sections, separated by a blank -line. The first section contains a number of headers, telling the client what -kind of data is following. Python code to generate a minimal header section -looks like this:: - - print("Content-Type: text/html") # HTML is following - print() # blank line, end of headers - -The second section is usually HTML, which allows the client software to display -nicely formatted text with header, in-line images, etc. Here's Python code that -prints a simple piece of HTML:: - - print("CGI script output") - print("

This is my first CGI script

") - print("Hello, world!") - - -.. _using-the-cgi-module: - -Using the cgi module --------------------- - -Begin by writing ``import cgi``. - -When you write a new script, consider adding these lines:: - - import cgitb - cgitb.enable() - -This activates a special exception handler that will display detailed reports in -the web browser if any errors occur. If you'd rather not show the guts of your -program to users of your script, you can have the reports saved to files -instead, with code like this:: - - import cgitb - cgitb.enable(display=0, logdir="/path/to/logdir") - -It's very helpful to use this feature during script development. The reports -produced by :mod:`cgitb` provide information that can save you a lot of time in -tracking down bugs. You can always remove the ``cgitb`` line later when you -have tested your script and are confident that it works correctly. - -To get at submitted form data, use the :class:`FieldStorage` class. If the form -contains non-ASCII characters, use the *encoding* keyword parameter set to the -value of the encoding defined for the document. It is usually contained in the -META tag in the HEAD section of the HTML document or by the -:mailheader:`Content-Type` header. This reads the form contents from the -standard input or the environment (depending on the value of various -environment variables set according to the CGI standard). Since it may consume -standard input, it should be instantiated only once. - -The :class:`FieldStorage` instance can be indexed like a Python dictionary. -It allows membership testing with the :keyword:`in` operator, and also supports -the standard dictionary method :meth:`~dict.keys` and the built-in function -:func:`len`. Form fields containing empty strings are ignored and do not appear -in the dictionary; to keep such values, provide a true value for the optional -*keep_blank_values* keyword parameter when creating the :class:`FieldStorage` -instance. - -For instance, the following code (which assumes that the -:mailheader:`Content-Type` header and blank line have already been printed) -checks that the fields ``name`` and ``addr`` are both set to a non-empty -string:: - - form = cgi.FieldStorage() - if "name" not in form or "addr" not in form: - print("

Error

") - print("Please fill in the name and addr fields.") - return - print("

name:", form["name"].value) - print("

addr:", form["addr"].value) - ...further form processing here... - -Here the fields, accessed through ``form[key]``, are themselves instances of -:class:`FieldStorage` (or :class:`MiniFieldStorage`, depending on the form -encoding). The :attr:`~FieldStorage.value` attribute of the instance yields -the string value of the field. The :meth:`~FieldStorage.getvalue` method -returns this string value directly; it also accepts an optional second argument -as a default to return if the requested key is not present. - -If the submitted form data contains more than one field with the same name, the -object retrieved by ``form[key]`` is not a :class:`FieldStorage` or -:class:`MiniFieldStorage` instance but a list of such instances. Similarly, in -this situation, ``form.getvalue(key)`` would return a list of strings. If you -expect this possibility (when your HTML form contains multiple fields with the -same name), use the :meth:`~FieldStorage.getlist` method, which always returns -a list of values (so that you do not need to special-case the single item -case). For example, this code concatenates any number of username fields, -separated by commas:: - - value = form.getlist("username") - usernames = ",".join(value) - -If a field represents an uploaded file, accessing the value via the -:attr:`~FieldStorage.value` attribute or the :meth:`~FieldStorage.getvalue` -method reads the entire file in memory as bytes. This may not be what you -want. You can test for an uploaded file by testing either the -:attr:`~FieldStorage.filename` attribute or the :attr:`~FieldStorage.file` -attribute. You can then read the data from the :attr:`!file` -attribute before it is automatically closed as part of the garbage collection of -the :class:`FieldStorage` instance -(the :func:`~io.RawIOBase.read` and :func:`~io.IOBase.readline` methods will -return bytes):: - - fileitem = form["userfile"] - if fileitem.file: - # It's an uploaded file; count lines - linecount = 0 - while True: - line = fileitem.file.readline() - if not line: break - linecount = linecount + 1 - -:class:`FieldStorage` objects also support being used in a :keyword:`with` -statement, which will automatically close them when done. - -If an error is encountered when obtaining the contents of an uploaded file -(for example, when the user interrupts the form submission by clicking on -a Back or Cancel button) the :attr:`~FieldStorage.done` attribute of the -object for the field will be set to the value -1. - -The file upload draft standard entertains the possibility of uploading multiple -files from one field (using a recursive :mimetype:`multipart/\*` encoding). -When this occurs, the item will be a dictionary-like :class:`FieldStorage` item. -This can be determined by testing its :attr:`!type` attribute, which should be -:mimetype:`multipart/form-data` (or perhaps another MIME type matching -:mimetype:`multipart/\*`). In this case, it can be iterated over recursively -just like the top-level form object. - -When a form is submitted in the "old" format (as the query string or as a single -data part of type :mimetype:`application/x-www-form-urlencoded`), the items will -actually be instances of the class :class:`MiniFieldStorage`. In this case, the -:attr:`!list`, :attr:`!file`, and :attr:`filename` attributes are always ``None``. - -A form submitted via POST that also has a query string will contain both -:class:`FieldStorage` and :class:`MiniFieldStorage` items. - -.. versionchanged:: 3.4 - The :attr:`~FieldStorage.file` attribute is automatically closed upon the - garbage collection of the creating :class:`FieldStorage` instance. - -.. versionchanged:: 3.5 - Added support for the context management protocol to the - :class:`FieldStorage` class. - - -Higher Level Interface ----------------------- - -The previous section explains how to read CGI form data using the -:class:`FieldStorage` class. This section describes a higher level interface -which was added to this class to allow one to do it in a more readable and -intuitive way. The interface doesn't make the techniques described in previous -sections obsolete --- they are still useful to process file uploads efficiently, -for example. - -.. XXX: Is this true ? - -The interface consists of two simple methods. Using the methods you can process -form data in a generic way, without the need to worry whether only one or more -values were posted under one name. - -In the previous section, you learned to write following code anytime you -expected a user to post more than one value under one name:: - - item = form.getvalue("item") - if isinstance(item, list): - # The user is requesting more than one item. - else: - # The user is requesting only one item. - -This situation is common for example when a form contains a group of multiple -checkboxes with the same name:: - - - - -In most situations, however, there's only one form control with a particular -name in a form and then you expect and need only one value associated with this -name. So you write a script containing for example this code:: - - user = form.getvalue("user").upper() - -The problem with the code is that you should never expect that a client will -provide valid input to your scripts. For example, if a curious user appends -another ``user=foo`` pair to the query string, then the script would crash, -because in this situation the ``getvalue("user")`` method call returns a list -instead of a string. Calling the :meth:`~str.upper` method on a list is not valid -(since lists do not have a method of this name) and results in an -:exc:`AttributeError` exception. - -Therefore, the appropriate way to read form data values was to always use the -code which checks whether the obtained value is a single value or a list of -values. That's annoying and leads to less readable scripts. - -A more convenient approach is to use the methods :meth:`~FieldStorage.getfirst` -and :meth:`~FieldStorage.getlist` provided by this higher level interface. - - -.. method:: FieldStorage.getfirst(name, default=None) - - This method always returns only one value associated with form field *name*. - The method returns only the first value in case that more values were posted - under such name. Please note that the order in which the values are received - may vary from browser to browser and should not be counted on. [#]_ If no such - form field or value exists then the method returns the value specified by the - optional parameter *default*. This parameter defaults to ``None`` if not - specified. - - -.. method:: FieldStorage.getlist(name) - - This method always returns a list of values associated with form field *name*. - The method returns an empty list if no such form field or value exists for - *name*. It returns a list consisting of one item if only one such value exists. - -Using these methods you can write nice compact code:: - - import cgi - form = cgi.FieldStorage() - user = form.getfirst("user", "").upper() # This way it's safe. - for item in form.getlist("item"): - do_something(item) - - -.. _functions-in-cgi-module: - -Functions ---------- - -These are useful if you want more control, or if you want to employ some of the -algorithms implemented in this module in other circumstances. - - -.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator="&") - - Parse a query in the environment or from a file (the file defaults to - ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are - passed to :func:`urllib.parse.parse_qs` unchanged. - - .. deprecated-removed:: 3.11 3.13 - This function, like the rest of the :mod:`cgi` module, is deprecated. - It can be replaced by calling :func:`urllib.parse.parse_qs` directly - on the desired query string (except for ``multipart/form-data`` input, - which can be handled as described for :func:`parse_multipart`). - - -.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator="&") - - Parse input of type :mimetype:`multipart/form-data` (for file uploads). - Arguments are *fp* for the input file, *pdict* for a dictionary containing - other parameters in the :mailheader:`Content-Type` header, and *encoding*, - the request encoding. - - Returns a dictionary just like :func:`urllib.parse.parse_qs`: keys are the - field names, each value is a list of values for that field. For non-file - fields, the value is a list of strings. - - This is easy to use but not much good if you are expecting megabytes to be - uploaded --- in that case, use the :class:`FieldStorage` class instead - which is much more flexible. - - .. versionchanged:: 3.7 - Added the *encoding* and *errors* parameters. For non-file fields, the - value is now a list of strings, not bytes. - - .. versionchanged:: 3.10 - Added the *separator* parameter. - - .. deprecated-removed:: 3.11 3.13 - This function, like the rest of the :mod:`cgi` module, is deprecated. - It can be replaced with the functionality in the :mod:`email` package - (e.g. :class:`email.message.EmailMessage`/:class:`email.message.Message`) - which implements the same MIME RFCs, or with the - `multipart `__ PyPI project. - - -.. function:: parse_header(string) - - Parse a MIME header (such as :mailheader:`Content-Type`) into a main value and a - dictionary of parameters. - - .. deprecated-removed:: 3.11 3.13 - This function, like the rest of the :mod:`cgi` module, is deprecated. - It can be replaced with the functionality in the :mod:`email` package, - which implements the same MIME RFCs. - - For example, with :class:`email.message.EmailMessage`:: - - from email.message import EmailMessage - msg = EmailMessage() - msg['content-type'] = 'application/json; charset="utf8"' - main, params = msg.get_content_type(), msg['content-type'].params - - -.. function:: test() - - Robust test CGI script, usable as main program. Writes minimal HTTP headers and - formats all information provided to the script in HTML format. - - -.. function:: print_environ() - - Format the shell environment in HTML. - - -.. function:: print_form(form) - - Format a form in HTML. - - -.. function:: print_directory() - - Format the current directory in HTML. - - -.. function:: print_environ_usage() - - Print a list of useful (used by CGI) environment variables in HTML. - - -.. _cgi-security: - -Caring about security ---------------------- - -.. index:: pair: CGI; security - -There's one important rule: if you invoke an external program (via -:func:`os.system`, :func:`os.popen` or other functions with similar -functionality), make very sure you don't pass arbitrary strings received from -the client to the shell. This is a well-known security hole whereby clever -hackers anywhere on the web can exploit a gullible CGI script to invoke -arbitrary shell commands. Even parts of the URL or field names cannot be -trusted, since the request doesn't have to come from your form! - -To be on the safe side, if you must pass a string gotten from a form to a shell -command, you should make sure the string contains only alphanumeric characters, -dashes, underscores, and periods. - - -Installing your CGI script on a Unix system -------------------------------------------- - -Read the documentation for your HTTP server and check with your local system -administrator to find the directory where CGI scripts should be installed; -usually this is in a directory :file:`cgi-bin` in the server tree. - -Make sure that your script is readable and executable by "others"; the Unix file -mode should be ``0o755`` octal (use ``chmod 0755 filename``). Make sure that the -first line of the script contains ``#!`` starting in column 1 followed by the -pathname of the Python interpreter, for instance:: - - #!/usr/local/bin/python - -Make sure the Python interpreter exists and is executable by "others". - -Make sure that any files your script needs to read or write are readable or -writable, respectively, by "others" --- their mode should be ``0o644`` for -readable and ``0o666`` for writable. This is because, for security reasons, the -HTTP server executes your script as user "nobody", without any special -privileges. It can only read (write, execute) files that everybody can read -(write, execute). The current directory at execution time is also different (it -is usually the server's cgi-bin directory) and the set of environment variables -is also different from what you get when you log in. In particular, don't count -on the shell's search path for executables (:envvar:`PATH`) or the Python module -search path (:envvar:`PYTHONPATH`) to be set to anything interesting. - -If you need to load modules from a directory which is not on Python's default -module search path, you can change the path in your script, before importing -other modules. For example:: - - import sys - sys.path.insert(0, "/usr/home/joe/lib/python") - sys.path.insert(0, "/usr/local/lib/python") - -(This way, the directory inserted last will be searched first!) - -Instructions for non-Unix systems will vary; check your HTTP server's -documentation (it will usually have a section on CGI scripts). - - -Testing your CGI script ------------------------ - -Unfortunately, a CGI script will generally not run when you try it from the -command line, and a script that works perfectly from the command line may fail -mysteriously when run from the server. There's one reason why you should still -test your script from the command line: if it contains a syntax error, the -Python interpreter won't execute it at all, and the HTTP server will most likely -send a cryptic error to the client. - -Assuming your script has no syntax errors, yet it does not work, you have no -choice but to read the next section. - - -Debugging CGI scripts ---------------------- - -.. index:: pair: CGI; debugging - -First of all, check for trivial installation errors --- reading the section -above on installing your CGI script carefully can save you a lot of time. If -you wonder whether you have understood the installation procedure correctly, try -installing a copy of this module file (:file:`cgi.py`) as a CGI script. When -invoked as a script, the file will dump its environment and the contents of the -form in HTML format. Give it the right mode etc., and send it a request. If it's -installed in the standard :file:`cgi-bin` directory, it should be possible to -send it a request by entering a URL into your browser of the form: - -.. code-block:: none - - http://yourhostname/cgi-bin/cgi.py?name=Joe+Blow&addr=At+Home - -If this gives an error of type 404, the server cannot find the script -- perhaps -you need to install it in a different directory. If it gives another error, -there's an installation problem that you should fix before trying to go any -further. If you get a nicely formatted listing of the environment and form -content (in this example, the fields should be listed as "addr" with value "At -Home" and "name" with value "Joe Blow"), the :file:`cgi.py` script has been -installed correctly. If you follow the same procedure for your own script, you -should now be able to debug it. - -The next step could be to call the :mod:`cgi` module's :func:`test` function -from your script: replace its main code with the single statement :: - - cgi.test() - -This should produce the same results as those gotten from installing the -:file:`cgi.py` file itself. - -When an ordinary Python script raises an unhandled exception (for whatever -reason: of a typo in a module name, a file that can't be opened, etc.), the -Python interpreter prints a nice traceback and exits. While the Python -interpreter will still do this when your CGI script raises an exception, most -likely the traceback will end up in one of the HTTP server's log files, or be -discarded altogether. - -Fortunately, once you have managed to get your script to execute *some* code, -you can easily send tracebacks to the web browser using the :mod:`cgitb` module. -If you haven't done so already, just add the lines:: - - import cgitb - cgitb.enable() - -to the top of your script. Then try running it again; when a problem occurs, -you should see a detailed report that will likely make apparent the cause of the -crash. - -If you suspect that there may be a problem in importing the :mod:`cgitb` module, -you can use an even more robust approach (which only uses built-in modules):: - - import sys - sys.stderr = sys.stdout - print("Content-Type: text/plain") - print() - ...your code here... - -This relies on the Python interpreter to print the traceback. The content type -of the output is set to plain text, which disables all HTML processing. If your -script works, the raw HTML will be displayed by your client. If it raises an -exception, most likely after the first two lines have been printed, a traceback -will be displayed. Because no HTML interpretation is going on, the traceback -will be readable. - - -Common problems and solutions ------------------------------ - -* Most HTTP servers buffer the output from CGI scripts until the script is - completed. This means that it is not possible to display a progress report on - the client's display while the script is running. - -* Check the installation instructions above. - -* Check the HTTP server's log files. (``tail -f logfile`` in a separate window - may be useful!) - -* Always check a script for syntax errors first, by doing something like - ``python script.py``. - -* If your script does not have any syntax errors, try adding ``import cgitb; - cgitb.enable()`` to the top of the script. - -* When invoking external programs, make sure they can be found. Usually, this - means using absolute path names --- :envvar:`PATH` is usually not set to a very - useful value in a CGI script. - -* When reading or writing external files, make sure they can be read or written - by the userid under which your CGI script will be running: this is typically the - userid under which the web server is running, or some explicitly specified - userid for a web server's ``suexec`` feature. - -* Don't try to give a CGI script a set-uid mode. This doesn't work on most - systems, and is a security liability as well. - -.. rubric:: Footnotes - -.. [#] Note that some recent versions of the HTML specification do state what - order the field values should be supplied in, but knowing whether a request - was received from a conforming browser, or even from a browser at all, is - tedious and error-prone. diff --git a/Doc/library/cgitb.rst b/Doc/library/cgitb.rst deleted file mode 100644 index 7f00bcd55c1e53..00000000000000 --- a/Doc/library/cgitb.rst +++ /dev/null @@ -1,89 +0,0 @@ -:mod:`cgitb` --- Traceback manager for CGI scripts -================================================== - -.. module:: cgitb - :synopsis: Configurable traceback handler for CGI scripts. - :deprecated: - -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/cgitb.py` - -.. index:: - single: CGI; exceptions - single: CGI; tracebacks - single: exceptions; in CGI scripts - single: tracebacks; in CGI scripts - -.. deprecated-removed:: 3.11 3.13 - The :mod:`cgitb` module is deprecated - (see :pep:`PEP 594 <594#cgitb>` for details). - --------------- - -The :mod:`cgitb` module provides a special exception handler for Python scripts. -(Its name is a bit misleading. It was originally designed to display extensive -traceback information in HTML for CGI scripts. It was later generalized to also -display this information in plain text.) After this module is activated, if an -uncaught exception occurs, a detailed, formatted report will be displayed. The -report includes a traceback showing excerpts of the source code for each level, -as well as the values of the arguments and local variables to currently running -functions, to help you debug the problem. Optionally, you can save this -information to a file instead of sending it to the browser. - -To enable this feature, simply add this to the top of your CGI script:: - - import cgitb - cgitb.enable() - -The options to the :func:`enable` function control whether the report is -displayed in the browser and whether the report is logged to a file for later -analysis. - - -.. function:: enable(display=1, logdir=None, context=5, format="html") - - .. index:: single: excepthook() (in module sys) - - This function causes the :mod:`cgitb` module to take over the interpreter's - default handling for exceptions by setting the value of :attr:`sys.excepthook`. - - The optional argument *display* defaults to ``1`` and can be set to ``0`` to - suppress sending the traceback to the browser. If the argument *logdir* is - present, the traceback reports are written to files. The value of *logdir* - should be a directory where these files will be placed. The optional argument - *context* is the number of lines of context to display around the current line - of source code in the traceback; this defaults to ``5``. If the optional - argument *format* is ``"html"``, the output is formatted as HTML. Any other - value forces plain text output. The default value is ``"html"``. - - -.. function:: text(info, context=5) - - This function handles the exception described by *info* (a 3-tuple containing - the result of :func:`sys.exc_info`), formatting its traceback as text and - returning the result as a string. The optional argument *context* is the - number of lines of context to display around the current line of source code - in the traceback; this defaults to ``5``. - - -.. function:: html(info, context=5) - - This function handles the exception described by *info* (a 3-tuple containing - the result of :func:`sys.exc_info`), formatting its traceback as HTML and - returning the result as a string. The optional argument *context* is the - number of lines of context to display around the current line of source code - in the traceback; this defaults to ``5``. - - -.. function:: handler(info=None) - - This function handles an exception using the default settings (that is, show a - report in the browser, but don't log to a file). This can be used when you've - caught an exception and want to report it using :mod:`cgitb`. The optional - *info* argument should be a 3-tuple containing an exception type, exception - value, and traceback object, exactly like the tuple returned by - :func:`sys.exc_info`. If the *info* argument is not supplied, the current - exception is obtained from :func:`sys.exc_info`. - diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst deleted file mode 100644 index 3b88e55b147882..00000000000000 --- a/Doc/library/chunk.rst +++ /dev/null @@ -1,142 +0,0 @@ -:mod:`chunk` --- Read IFF chunked data -====================================== - -.. module:: chunk - :synopsis: Module to read IFF chunks. - :deprecated: - -.. moduleauthor:: Sjoerd Mullender -.. sectionauthor:: Sjoerd Mullender - -**Source code:** :source:`Lib/chunk.py` - -.. index:: - single: Audio Interchange File Format - single: AIFF - single: AIFF-C - single: Real Media File Format - single: RMFF - -.. deprecated-removed:: 3.11 3.13 - The :mod:`chunk` module is deprecated - (see :pep:`PEP 594 <594#chunk>` for details). - --------------- - -This module provides an interface for reading files that use EA IFF 85 chunks. -[#]_ This format is used in at least the Audio Interchange File Format -(AIFF/AIFF-C) and the Real Media File Format (RMFF). The WAVE audio file format -is closely related and can also be read using this module. - -A chunk has the following structure: - -+---------+--------+-------------------------------+ -| Offset | Length | Contents | -+=========+========+===============================+ -| 0 | 4 | Chunk ID | -+---------+--------+-------------------------------+ -| 4 | 4 | Size of chunk in big-endian | -| | | byte order, not including the | -| | | header | -+---------+--------+-------------------------------+ -| 8 | *n* | Data bytes, where *n* is the | -| | | size given in the preceding | -| | | field | -+---------+--------+-------------------------------+ -| 8 + *n* | 0 or 1 | Pad byte needed if *n* is odd | -| | | and chunk alignment is used | -+---------+--------+-------------------------------+ - -The ID is a 4-byte string which identifies the type of chunk. - -The size field (a 32-bit value, encoded using big-endian byte order) gives the -size of the chunk data, not including the 8-byte header. - -Usually an IFF-type file consists of one or more chunks. The proposed usage of -the :class:`Chunk` class defined here is to instantiate an instance at the start -of each chunk and read from the instance until it reaches the end, after which a -new instance can be instantiated. At the end of the file, creating a new -instance will fail with an :exc:`EOFError` exception. - - -.. class:: Chunk(file, align=True, bigendian=True, inclheader=False) - - Class which represents a chunk. The *file* argument is expected to be a - file-like object. An instance of this class is specifically allowed. The - only method that is needed is :meth:`~io.IOBase.read`. If the methods - :meth:`~io.IOBase.seek` and :meth:`~io.IOBase.tell` are present and don't - raise an exception, they are also used. - If these methods are present and raise an exception, they are expected to not - have altered the object. If the optional argument *align* is true, chunks - are assumed to be aligned on 2-byte boundaries. If *align* is false, no - alignment is assumed. The default value is true. If the optional argument - *bigendian* is false, the chunk size is assumed to be in little-endian order. - This is needed for WAVE audio files. The default value is true. If the - optional argument *inclheader* is true, the size given in the chunk header - includes the size of the header. The default value is false. - - A :class:`Chunk` object supports the following methods: - - - .. method:: getname() - - Returns the name (ID) of the chunk. This is the first 4 bytes of the - chunk. - - - .. method:: getsize() - - Returns the size of the chunk. - - - .. method:: close() - - Close and skip to the end of the chunk. This does not close the - underlying file. - - The remaining methods will raise :exc:`OSError` if called after the - :meth:`close` method has been called. Before Python 3.3, they used to - raise :exc:`IOError`, now an alias of :exc:`OSError`. - - - .. method:: isatty() - - Returns ``False``. - - - .. method:: seek(pos, whence=0) - - Set the chunk's current position. The *whence* argument is optional and - defaults to ``0`` (absolute file positioning); other values are ``1`` - (seek relative to the current position) and ``2`` (seek relative to the - file's end). There is no return value. If the underlying file does not - allow seek, only forward seeks are allowed. - - - .. method:: tell() - - Return the current position into the chunk. - - - .. method:: read(size=-1) - - Read at most *size* bytes from the chunk (less if the read hits the end of - the chunk before obtaining *size* bytes). If the *size* argument is - negative or omitted, read all data until the end of the chunk. An empty - bytes object is returned when the end of the chunk is encountered - immediately. - - - .. method:: skip() - - Skip to the end of the chunk. All further calls to :meth:`read` for the - chunk will return ``b''``. If you are not interested in the contents of - the chunk, this method should be called so that the file points to the - start of the next chunk. - - -.. rubric:: Footnotes - -.. [#] "EA IFF 85" Standard for Interchange Format Files, Jerry Morrison, Electronic - Arts, January 1985. - diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index b17d58e1cc0ce1..fdac51d9603ceb 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -9,7 +9,7 @@ This module provides access to mathematical functions for complex numbers. The functions in this module accept integers, floating-point numbers or complex numbers as arguments. They will also accept any Python object that has either a -:meth:`__complex__` or a :meth:`__float__` method: these methods are used to +:meth:`~object.__complex__` or a :meth:`~object.__float__` method: these methods are used to convert the object to a complex or floating-point number, respectively, and the function is then applied to the result of the conversion. diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index fd5df96dfd0b3d..1318ffe5a48d53 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -26,6 +26,13 @@ interface. key; it defaults to :kbd:`Tab`. If *completekey* is not :const:`None` and :mod:`readline` is available, command completion is done automatically. + The default, ``'tab'``, is treated specially, so that it refers to the + :kbd:`Tab` key on every :data:`readline.backend`. + Specifically, if :data:`readline.backend` is ``editline``, + ``Cmd`` will use ``'^I'`` instead of ``'tab'``. + Note that other values are not treated this way, and might only work + with a specific backend. + The optional arguments *stdin* and *stdout* specify the input and output file objects that the Cmd instance or subclass instance will use for input and output. If not specified, they will default to :data:`sys.stdin` and @@ -35,6 +42,9 @@ interface. :attr:`use_rawinput` attribute to ``False``, otherwise *stdin* will be ignored. + .. versionchanged:: 3.13 + ``completekey='tab'`` is replaced by ``'^I'`` for ``editline``. + .. _cmd-objects: diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst new file mode 100644 index 00000000000000..b2379befeffcba --- /dev/null +++ b/Doc/library/cmdline.rst @@ -0,0 +1,57 @@ +++++++++++++++++++++++++++++++++++++ +Modules command-line interface (CLI) +++++++++++++++++++++++++++++++++++++ + +The following modules have a command-line interface. + +* :ref:`ast ` +* :ref:`asyncio ` +* :mod:`base64` +* :ref:`calendar ` +* :mod:`code` +* :ref:`compileall ` +* :mod:`cProfile`: see :ref:`profile ` +* :ref:`difflib ` +* :ref:`dis ` +* :mod:`doctest` +* :mod:`!encodings.rot_13` +* :mod:`ensurepip` +* :mod:`filecmp` +* :mod:`fileinput` +* :mod:`ftplib` +* :ref:`gzip ` +* :ref:`http.server ` +* :mod:`!idlelib` +* :ref:`inspect ` +* :ref:`json.tool ` +* :mod:`mimetypes` +* :mod:`pdb` +* :mod:`pickle` +* :ref:`pickletools ` +* :mod:`platform` +* :mod:`poplib` +* :ref:`profile ` +* :mod:`pstats` +* :ref:`py_compile ` +* :mod:`pyclbr` +* :mod:`pydoc` +* :mod:`quopri` +* :mod:`runpy` +* :ref:`site ` +* :ref:`sqlite3 ` +* :ref:`sysconfig ` +* :mod:`tabnanny` +* :ref:`tarfile ` +* :mod:`!this` +* :ref:`timeit ` +* :ref:`tokenize ` +* :ref:`trace ` +* :mod:`turtledemo` +* :ref:`unittest ` +* :ref:`uuid ` +* :mod:`venv` +* :mod:`webbrowser` +* :ref:`zipapp ` +* :ref:`zipfile ` + +See also the :ref:`Python command-line interface `. diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 538e5afc7822aa..091840781bd235 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -23,20 +23,25 @@ build applications which provide an interactive interpreter prompt. ``'__doc__'`` set to ``None``. -.. class:: InteractiveConsole(locals=None, filename="") +.. class:: InteractiveConsole(locals=None, filename="", local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar - ``sys.ps1`` and ``sys.ps2``, and input buffering. + ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is True, + ``exit()`` and ``quit()`` in the console will not raise :exc:`SystemExit`, but + instead return to the calling code. + .. versionchanged:: 3.13 + Added *local_exit* parameter. -.. function:: interact(banner=None, readfunc=None, local=None, exitmsg=None) +.. function:: interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False) Convenience function to run a read-eval-print loop. This creates a new instance of :class:`InteractiveConsole` and sets *readfunc* to be used as the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is provided, it is passed to the :class:`InteractiveConsole` constructor for - use as the default namespace for the interpreter loop. The :meth:`interact` + use as the default namespace for the interpreter loop. If *local_exit* is provided, + it is passed to the :class:`InteractiveConsole` constructor. The :meth:`interact` method of the instance is then run with *banner* and *exitmsg* passed as the banner and exit message to use, if provided. The console object is discarded after use. @@ -44,6 +49,8 @@ build applications which provide an interactive interpreter prompt. .. versionchanged:: 3.6 Added *exitmsg* parameter. + .. versionchanged:: 3.13 + Added *local_exit* parameter. .. function:: compile_command(source, filename="", symbol="single") @@ -163,12 +170,12 @@ interpreter objects as well as the following additions. Push a line of source text to the interpreter. The line should not have a trailing newline; it may have internal newlines. The line is appended to a - buffer and the interpreter's :meth:`runsource` method is called with the + buffer and the interpreter's :meth:`~InteractiveInterpreter.runsource` method is called with the concatenated contents of the buffer as source. If this indicates that the command was executed or invalid, the buffer is reset; otherwise, the command is incomplete, and the buffer is left as it was after the line was appended. The return value is ``True`` if more input is required, ``False`` if the line was - dealt with in some way (this is the same as :meth:`runsource`). + dealt with in some way (this is the same as :meth:`!runsource`). .. method:: InteractiveConsole.resetbuffer() diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 8225236350d22e..4617624686b1f3 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -345,9 +345,10 @@ The following error handlers can be used with all Python +-------------------------+-----------------------------------------------+ | ``'backslashreplace'`` | Replace with backslashed escape sequences. | | | On encoding, use hexadecimal form of Unicode | -| | code point with formats ``\xhh`` ``\uxxxx`` | -| | ``\Uxxxxxxxx``. On decoding, use hexadecimal | -| | form of byte value with format ``\xhh``. | +| | code point with formats :samp:`\\x{hh}` | +| | :samp:`\\u{xxxx}` :samp:`\\U{xxxxxxxx}`. | +| | On decoding, use hexadecimal form of byte | +| | value with format :samp:`\\x{hh}`. | | | Implemented in | | | :func:`backslashreplace_errors`. | +-------------------------+-----------------------------------------------+ @@ -373,8 +374,9 @@ The following error handlers are only applicable to encoding (within +=========================+===============================================+ | ``'xmlcharrefreplace'`` | Replace with XML/HTML numeric character | | | reference, which is a decimal form of Unicode | -| | code point with format ``&#num;`` Implemented | -| | in :func:`xmlcharrefreplace_errors`. | +| | code point with format :samp:`&#{num};`. | +| | Implemented in | +| | :func:`xmlcharrefreplace_errors`. | +-------------------------+-----------------------------------------------+ | ``'namereplace'`` | Replace with ``\N{...}`` escape sequences, | | | what appears in the braces is the Name | @@ -478,8 +480,9 @@ functions: Malformed data is replaced by a backslashed escape sequence. On encoding, use the hexadecimal form of Unicode code point with formats - ``\xhh`` ``\uxxxx`` ``\Uxxxxxxxx``. On decoding, use the hexadecimal form of - byte value with format ``\xhh``. + :samp:`\\x{hh}` :samp:`\\u{xxxx}` :samp:`\\U{xxxxxxxx}`. + On decoding, use the hexadecimal form of + byte value with format :samp:`\\x{hh}`. .. versionchanged:: 3.5 Works with decoding and translating. @@ -492,7 +495,7 @@ functions: The unencodable character is replaced by an appropriate XML/HTML numeric character reference, which is a decimal form of Unicode code point with - format ``&#num;`` . + format :samp:`&#{num};` . .. function:: namereplace_errors(exception) @@ -517,44 +520,46 @@ The base :class:`Codec` class defines these methods which also define the function interfaces of the stateless encoder and decoder: -.. method:: Codec.encode(input, errors='strict') +.. class:: Codec + + .. method:: encode(input, errors='strict') - Encodes the object *input* and returns a tuple (output object, length consumed). - For instance, :term:`text encoding` converts - a string object to a bytes object using a particular - character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). + Encodes the object *input* and returns a tuple (output object, length consumed). + For instance, :term:`text encoding` converts + a string object to a bytes object using a particular + character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. + The *errors* argument defines the error handling to apply. + It defaults to ``'strict'`` handling. - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamWriter` for codecs which have to keep state in order to make - encoding efficient. + The method may not store state in the :class:`Codec` instance. Use + :class:`StreamWriter` for codecs which have to keep state in order to make + encoding efficient. - The encoder must be able to handle zero length input and return an empty object - of the output object type in this situation. + The encoder must be able to handle zero length input and return an empty object + of the output object type in this situation. -.. method:: Codec.decode(input, errors='strict') + .. method:: decode(input, errors='strict') - Decodes the object *input* and returns a tuple (output object, length - consumed). For instance, for a :term:`text encoding`, decoding converts - a bytes object encoded using a particular - character set encoding to a string object. + Decodes the object *input* and returns a tuple (output object, length + consumed). For instance, for a :term:`text encoding`, decoding converts + a bytes object encoded using a particular + character set encoding to a string object. - For text encodings and bytes-to-bytes codecs, - *input* must be a bytes object or one which provides the read-only - buffer interface -- for example, buffer objects and memory mapped files. + For text encodings and bytes-to-bytes codecs, + *input* must be a bytes object or one which provides the read-only + buffer interface -- for example, buffer objects and memory mapped files. - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. + The *errors* argument defines the error handling to apply. + It defaults to ``'strict'`` handling. - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamReader` for codecs which have to keep state in order to make - decoding efficient. + The method may not store state in the :class:`Codec` instance. Use + :class:`StreamReader` for codecs which have to keep state in order to make + decoding efficient. - The decoder must be able to handle zero length input and return an empty object - of the output object type in this situation. + The decoder must be able to handle zero length input and return an empty object + of the output object type in this situation. Incremental Encoding and Decoding @@ -702,7 +707,7 @@ Stream Encoding and Decoding The :class:`StreamWriter` and :class:`StreamReader` classes provide generic working interfaces which can be used to implement new encoding submodules very -easily. See :mod:`encodings.utf_8` for an example of how this is done. +easily. See :mod:`!encodings.utf_8` for an example of how this is done. .. _stream-writer-objects: @@ -892,9 +897,10 @@ The design is such that one can use the factory functions returned by the .. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors='strict') Creates a :class:`StreamRecoder` instance which implements a two-way conversion: - *encode* and *decode* work on the frontend — the data visible to - code calling :meth:`read` and :meth:`write`, while *Reader* and *Writer* - work on the backend — the data in *stream*. + *encode* and *decode* work on the frontend — the data visible to + code calling :meth:`~StreamReader.read` and :meth:`~StreamWriter.write`, + while *Reader* and *Writer* + work on the backend — the data in *stream*. You can use these objects to do transparent transcodings, e.g., from Latin-1 to UTF-8 and back. @@ -1346,9 +1352,10 @@ encodings. | | | supported. | +--------------------+---------+---------------------------+ | raw_unicode_escape | | Latin-1 encoding with | -| | | ``\uXXXX`` and | -| | | ``\UXXXXXXXX`` for other | -| | | code points. Existing | +| | | :samp:`\\u{XXXX}` and | +| | | :samp:`\\U{XXXXXXXX}` | +| | | for other code points. | +| | | Existing | | | | backslashes are not | | | | escaped in any way. | | | | It is used in the Python | @@ -1413,8 +1420,8 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` | | quotedprintable, | quoted printable. | ``quotetabs=True`` / | | | quoted_printable | | :meth:`quopri.decode` | +----------------------+------------------+------------------------------+------------------------------+ -| uu_codec | uu | Convert the operand using | :meth:`uu.encode` / | -| | | uuencode. | :meth:`uu.decode` | +| uu_codec | uu | Convert the operand using | | +| | | uuencode. | | +----------------------+------------------+------------------------------+------------------------------+ | zlib_codec | zip, zlib | Compress the operand using | :meth:`zlib.compress` / | | | | gzip. | :meth:`zlib.decompress` | diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 90df499f8207b7..55606e1c5f09ac 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -58,7 +58,7 @@ To do just the former: .. class:: Compile() - Instances of this class have :meth:`__call__` methods identical in signature to + Instances of this class have :meth:`~object.__call__` methods identical in signature to the built-in function :func:`compile`, but with the difference that if the instance compiles program text containing a :mod:`__future__` statement, the instance 'remembers' and compiles all subsequent program texts with the @@ -67,7 +67,7 @@ To do just the former: .. class:: CommandCompiler() - Instances of this class have :meth:`__call__` methods identical in signature to + Instances of this class have :meth:`~object.__call__` methods identical in signature to :func:`compile_command`; the difference is that if the instance compiles program text containing a :mod:`__future__` statement, the instance 'remembers' and compiles all subsequent program texts with the statement in force. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 43a3286ba832cf..edc078953290d7 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -192,7 +192,7 @@ ABC Inherits from Abstract Methods Mi .. [2] Checking ``isinstance(obj, Iterable)`` detects classes that are registered as :class:`Iterable` or that have an :meth:`__iter__` method, but it does not detect classes that iterate with the - :meth:`__getitem__` method. The only reliable way to determine + :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -222,7 +222,7 @@ Collections Abstract Base Classes -- Detailed Descriptions Checking ``isinstance(obj, Iterable)`` detects classes that are registered as :class:`Iterable` or that have an :meth:`__iter__` method, but it does - not detect classes that iterate with the :meth:`__getitem__` method. + not detect classes that iterate with the :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -262,8 +262,8 @@ Collections Abstract Base Classes -- Detailed Descriptions Implementation note: Some of the mixin methods, such as :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`__getitem__` method. - Consequently, if :meth:`__getitem__` is implemented with constant + repeated calls to the underlying :meth:`~object.__getitem__` method. + Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; however, if the underlying method is linear (as it would be with a linked list), the mixins will have quadratic performance and will diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index bb46782c06e1c8..233b2c6a771f4a 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -120,26 +120,26 @@ The class can be used to simulate nested scopes and is useful in templating. .. seealso:: - * The `MultiContext class - `_ - in the Enthought `CodeTools package - `_ has options to support - writing to any mapping in the chain. + * The `MultiContext class + `_ + in the Enthought `CodeTools package + `_ has options to support + writing to any mapping in the chain. - * Django's `Context class - `_ - for templating is a read-only chain of mappings. It also features - pushing and popping of contexts similar to the - :meth:`~collections.ChainMap.new_child` method and the - :attr:`~collections.ChainMap.parents` property. + * Django's `Context class + `_ + for templating is a read-only chain of mappings. It also features + pushing and popping of contexts similar to the + :meth:`~collections.ChainMap.new_child` method and the + :attr:`~collections.ChainMap.parents` property. - * The `Nested Contexts recipe - `_ has options to control - whether writes and other mutations apply only to the first mapping or to - any mapping in the chain. + * The `Nested Contexts recipe + `_ has options to control + whether writes and other mutations apply only to the first mapping or to + any mapping in the chain. - * A `greatly simplified read-only version of Chainmap - `_. + * A `greatly simplified read-only version of Chainmap + `_. :class:`ChainMap` Examples and Recipes @@ -358,7 +358,7 @@ Common patterns for working with :class:`Counter` objects:: list(c) # list unique elements set(c) # convert to a set dict(c) # convert to a regular dictionary - c.items() # convert to a list of (elem, cnt) pairs + c.items() # access the (elem, cnt) pairs Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs c.most_common()[:-n-1:-1] # n least common elements +c # remove zero and negative counts @@ -429,22 +429,22 @@ or subtracting from an empty counter. .. seealso:: - * `Bag class `_ - in Smalltalk. + * `Bag class `_ + in Smalltalk. - * Wikipedia entry for `Multisets `_. + * Wikipedia entry for `Multisets `_. - * `C++ multisets `_ - tutorial with examples. + * `C++ multisets `_ + tutorial with examples. - * For mathematical operations on multisets and their use cases, see - *Knuth, Donald. The Art of Computer Programming Volume II, - Section 4.6.3, Exercise 19*. + * For mathematical operations on multisets and their use cases, see + *Knuth, Donald. The Art of Computer Programming Volume II, + Section 4.6.3, Exercise 19*. - * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`:: + * To enumerate all distinct multisets of a given size over a given set of + elements, see :func:`itertools.combinations_with_replacement`:: - map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC + map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC :class:`deque` objects @@ -743,12 +743,12 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, If calling :attr:`default_factory` raises an exception this exception is propagated unchanged. - This method is called by the :meth:`__getitem__` method of the + This method is called by the :meth:`~object.__getitem__` method of the :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`__getitem__`. + returns or raises is then returned or raised by :meth:`~object.__getitem__`. Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`__getitem__`. This means that :meth:`get` will, like normal + :meth:`~object.__getitem__`. This means that :meth:`get` will, like normal dictionaries, return ``None`` as a default rather than using :attr:`default_factory`. @@ -979,6 +979,12 @@ field names, the method and attribute names start with an underscore. >>> for partnum, record in inventory.items(): ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) + Named tuples are also supported by generic function :func:`copy.replace`. + + .. versionchanged:: 3.13 + Raise :exc:`TypeError` instead of :exc:`ValueError` for invalid + keyword arguments. + .. attribute:: somenamedtuple._fields Tuple of strings listing the field names. Useful for introspection @@ -1060,20 +1066,20 @@ fields: .. seealso:: - * See :class:`typing.NamedTuple` for a way to add type hints for named - tuples. It also provides an elegant notation using the :keyword:`class` - keyword:: + * See :class:`typing.NamedTuple` for a way to add type hints for named + tuples. It also provides an elegant notation using the :keyword:`class` + keyword:: - class Component(NamedTuple): - part_number: int - weight: float - description: Optional[str] = None + class Component(NamedTuple): + part_number: int + weight: float + description: Optional[str] = None - * See :meth:`types.SimpleNamespace` for a mutable namespace based on an - underlying dictionary instead of a tuple. + * See :meth:`types.SimpleNamespace` for a mutable namespace based on an + underlying dictionary instead of a tuple. - * The :mod:`dataclasses` module provides a decorator and functions for - automatically adding generated special methods to user-defined classes. + * The :mod:`dataclasses` module provides a decorator and functions for + automatically adding generated special methods to user-defined classes. :class:`OrderedDict` objects @@ -1224,7 +1230,7 @@ variants of :func:`functools.lru_cache`: result = self.func(*args) self.cache[args] = time(), result if len(self.cache) > self.maxsize: - self.cache.popitem(0) + self.cache.popitem(last=False) return result @@ -1256,12 +1262,12 @@ variants of :func:`functools.lru_cache`: if self.requests[args] <= self.cache_after: self.requests.move_to_end(args) if len(self.requests) > self.maxrequests: - self.requests.popitem(0) + self.requests.popitem(last=False) else: self.requests.pop(args, None) self.cache[args] = result if len(self.cache) > self.maxsize: - self.cache.popitem(0) + self.cache.popitem(last=False) return result .. doctest:: diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 180f5b81c2b615..df1eefab839cc1 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -16,6 +16,8 @@ have write permission to the library directories. .. include:: ../includes/wasm-notavail.rst +.. _compileall-cli: + Command-line use ---------------- @@ -24,28 +26,28 @@ compile Python sources. .. program:: compileall -.. cmdoption:: directory ... - file ... +.. option:: directory ... + file ... Positional arguments are files to compile or directories that contain source files, traversed recursively. If no argument is given, behave as if - the command line was ``-l ``. + the command line was :samp:`-l {}`. -.. cmdoption:: -l +.. option:: -l Do not recurse into subdirectories, only compile source code files directly contained in the named or implied directories. -.. cmdoption:: -f +.. option:: -f Force rebuild even if timestamps are up-to-date. -.. cmdoption:: -q +.. option:: -q Do not print the list of files compiled. If passed once, error messages will still be printed. If passed twice (``-qq``), all output is suppressed. -.. cmdoption:: -d destdir +.. option:: -d destdir Directory prepended to the path to each file being compiled. This will appear in compilation time tracebacks, and is also compiled in to the @@ -53,45 +55,45 @@ compile Python sources. cases where the source file does not exist at the time the byte-code file is executed. -.. cmdoption:: -s strip_prefix -.. cmdoption:: -p prepend_prefix +.. option:: -s strip_prefix +.. option:: -p prepend_prefix Remove (``-s``) or append (``-p``) the given prefix of paths recorded in the ``.pyc`` files. Cannot be combined with ``-d``. -.. cmdoption:: -x regex +.. option:: -x regex regex is used to search the full path to each file considered for compilation, and if the regex produces a match, the file is skipped. -.. cmdoption:: -i list +.. option:: -i list Read the file ``list`` and add each line that it contains to the list of files and directories to compile. If ``list`` is ``-``, read lines from ``stdin``. -.. cmdoption:: -b +.. option:: -b Write the byte-code files to their legacy locations and names, which may overwrite byte-code files created by another version of Python. The default is to write files to their :pep:`3147` locations and names, which allows byte-code files from multiple versions of Python to coexist. -.. cmdoption:: -r +.. option:: -r Control the maximum recursion level for subdirectories. If this is given, then ``-l`` option will not be taken into account. :program:`python -m compileall -r 0` is equivalent to :program:`python -m compileall -l`. -.. cmdoption:: -j N +.. option:: -j N Use *N* workers to compile the files within the given directory. - If ``0`` is used, then the result of :func:`os.cpu_count()` + If ``0`` is used, then the result of :func:`os.process_cpu_count()` will be used. -.. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] +.. option:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] Control how the generated byte-code files are invalidated at runtime. The ``timestamp`` value, means that ``.pyc`` files with the source timestamp @@ -104,17 +106,17 @@ compile Python sources. variable is not set, and ``checked-hash`` if the ``SOURCE_DATE_EPOCH`` environment variable is set. -.. cmdoption:: -o level +.. option:: -o level Compile with the given optimization level. May be used multiple times to compile for multiple levels at a time (for example, ``compileall -o 1 -o 2``). -.. cmdoption:: -e dir +.. option:: -e dir Ignore symlinks pointing outside the given directory. -.. cmdoption:: --hardlink-dupes +.. option:: --hardlink-dupes If two ``.pyc`` files with different optimization level have the same content, use hard links to consolidate duplicate files. @@ -141,9 +143,9 @@ There is no command-line option to control the optimization level used by the :func:`compile` function, because the Python interpreter itself already provides the option: :program:`python -O -m compileall`. -Similarly, the :func:`compile` function respects the :attr:`sys.pycache_prefix` +Similarly, the :func:`compile` function respects the :data:`sys.pycache_prefix` setting. The generated bytecode cache will only be useful if :func:`compile` is -run with the same :attr:`sys.pycache_prefix` (if any) that will be used at +run with the same :data:`sys.pycache_prefix` (if any) that will be used at runtime. Public functions diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 09c9fc4e6e227a..deefb8606ead84 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -29,83 +29,83 @@ Executor Objects An abstract class that provides methods to execute calls asynchronously. It should not be used directly, but through its concrete subclasses. - .. method:: submit(fn, /, *args, **kwargs) + .. method:: submit(fn, /, *args, **kwargs) - Schedules the callable, *fn*, to be executed as ``fn(*args, **kwargs)`` - and returns a :class:`Future` object representing the execution of the - callable. :: + Schedules the callable, *fn*, to be executed as ``fn(*args, **kwargs)`` + and returns a :class:`Future` object representing the execution of the + callable. :: - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(pow, 323, 1235) - print(future.result()) + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(pow, 323, 1235) + print(future.result()) - .. method:: map(func, *iterables, timeout=None, chunksize=1) + .. method:: map(func, *iterables, timeout=None, chunksize=1) - Similar to :func:`map(func, *iterables) ` except: + Similar to :func:`map(func, *iterables) ` except: - * the *iterables* are collected immediately rather than lazily; + * the *iterables* are collected immediately rather than lazily; - * *func* is executed asynchronously and several calls to - *func* may be made concurrently. + * *func* is executed asynchronously and several calls to + *func* may be made concurrently. - The returned iterator raises a :exc:`TimeoutError` - if :meth:`~iterator.__next__` is called and the result isn't available - after *timeout* seconds from the original call to :meth:`Executor.map`. - *timeout* can be an int or a float. If *timeout* is not specified or - ``None``, there is no limit to the wait time. + The returned iterator raises a :exc:`TimeoutError` + if :meth:`~iterator.__next__` is called and the result isn't available + after *timeout* seconds from the original call to :meth:`Executor.map`. + *timeout* can be an int or a float. If *timeout* is not specified or + ``None``, there is no limit to the wait time. - If a *func* call raises an exception, then that exception will be - raised when its value is retrieved from the iterator. + If a *func* call raises an exception, then that exception will be + raised when its value is retrieved from the iterator. - When using :class:`ProcessPoolExecutor`, this method chops *iterables* - into a number of chunks which it submits to the pool as separate - tasks. The (approximate) size of these chunks can be specified by - setting *chunksize* to a positive integer. For very long iterables, - using a large value for *chunksize* can significantly improve - performance compared to the default size of 1. With - :class:`ThreadPoolExecutor`, *chunksize* has no effect. + When using :class:`ProcessPoolExecutor`, this method chops *iterables* + into a number of chunks which it submits to the pool as separate + tasks. The (approximate) size of these chunks can be specified by + setting *chunksize* to a positive integer. For very long iterables, + using a large value for *chunksize* can significantly improve + performance compared to the default size of 1. With + :class:`ThreadPoolExecutor`, *chunksize* has no effect. - .. versionchanged:: 3.5 - Added the *chunksize* argument. + .. versionchanged:: 3.5 + Added the *chunksize* argument. - .. method:: shutdown(wait=True, *, cancel_futures=False) + .. method:: shutdown(wait=True, *, cancel_futures=False) - Signal the executor that it should free any resources that it is using - when the currently pending futures are done executing. Calls to - :meth:`Executor.submit` and :meth:`Executor.map` made after shutdown will - raise :exc:`RuntimeError`. + Signal the executor that it should free any resources that it is using + when the currently pending futures are done executing. Calls to + :meth:`Executor.submit` and :meth:`Executor.map` made after shutdown will + raise :exc:`RuntimeError`. - If *wait* is ``True`` then this method will not return until all the - pending futures are done executing and the resources associated with the - executor have been freed. If *wait* is ``False`` then this method will - return immediately and the resources associated with the executor will be - freed when all pending futures are done executing. Regardless of the - value of *wait*, the entire Python program will not exit until all - pending futures are done executing. + If *wait* is ``True`` then this method will not return until all the + pending futures are done executing and the resources associated with the + executor have been freed. If *wait* is ``False`` then this method will + return immediately and the resources associated with the executor will be + freed when all pending futures are done executing. Regardless of the + value of *wait*, the entire Python program will not exit until all + pending futures are done executing. - If *cancel_futures* is ``True``, this method will cancel all pending - futures that the executor has not started running. Any futures that - are completed or running won't be cancelled, regardless of the value - of *cancel_futures*. + If *cancel_futures* is ``True``, this method will cancel all pending + futures that the executor has not started running. Any futures that + are completed or running won't be cancelled, regardless of the value + of *cancel_futures*. - If both *cancel_futures* and *wait* are ``True``, all futures that the - executor has started running will be completed prior to this method - returning. The remaining futures are cancelled. + If both *cancel_futures* and *wait* are ``True``, all futures that the + executor has started running will be completed prior to this method + returning. The remaining futures are cancelled. - You can avoid having to call this method explicitly if you use the - :keyword:`with` statement, which will shutdown the :class:`Executor` - (waiting as if :meth:`Executor.shutdown` were called with *wait* set to - ``True``):: + You can avoid having to call this method explicitly if you use the + :keyword:`with` statement, which will shutdown the :class:`Executor` + (waiting as if :meth:`Executor.shutdown` were called with *wait* set to + ``True``):: - import shutil - with ThreadPoolExecutor(max_workers=4) as e: - e.submit(shutil.copy, 'src1.txt', 'dest1.txt') - e.submit(shutil.copy, 'src2.txt', 'dest2.txt') - e.submit(shutil.copy, 'src3.txt', 'dest3.txt') - e.submit(shutil.copy, 'src4.txt', 'dest4.txt') + import shutil + with ThreadPoolExecutor(max_workers=4) as e: + e.submit(shutil.copy, 'src1.txt', 'dest1.txt') + e.submit(shutil.copy, 'src2.txt', 'dest2.txt') + e.submit(shutil.copy, 'src3.txt', 'dest3.txt') + e.submit(shutil.copy, 'src4.txt', 'dest4.txt') - .. versionchanged:: 3.9 - Added *cancel_futures*. + .. versionchanged:: 3.9 + Added *cancel_futures*. ThreadPoolExecutor @@ -188,6 +188,10 @@ And:: ThreadPoolExecutor now reuses idle worker threads before starting *max_workers* worker threads too. + .. versionchanged:: 3.13 + Default value of *max_workers* is changed to + ``min(32, (os.process_cpu_count() or 1) + 4)``. + .. _threadpoolexecutor-example: @@ -243,7 +247,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. An :class:`Executor` subclass that executes calls asynchronously using a pool of at most *max_workers* processes. If *max_workers* is ``None`` or not - given, it will default to the number of processors on the machine. + given, it will default to :func:`os.process_cpu_count`. If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError` will be raised. On Windows, *max_workers* must be less than or equal to ``61``. If it is not @@ -293,6 +297,18 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. The *max_tasks_per_child* argument was added to allow users to control the lifetime of workers in the pool. + .. versionchanged:: 3.12 + On POSIX systems, if your application has multiple threads and the + :mod:`multiprocessing` context uses the ``"fork"`` start method: + The :func:`os.fork` function called internally to spawn workers may raise a + :exc:`DeprecationWarning`. Pass a *mp_context* configured to use a + different start method. See the :func:`os.fork` documentation for + further explanation. + + .. versionchanged:: 3.13 + *max_workers* uses :func:`os.process_cpu_count` by default, instead of + :func:`os.cpu_count`. + .. _processpoolexecutor-example: ProcessPoolExecutor Example @@ -345,117 +361,117 @@ The :class:`Future` class encapsulates the asynchronous execution of a callable. instances are created by :meth:`Executor.submit` and should not be created directly except for testing. - .. method:: cancel() + .. method:: cancel() - Attempt to cancel the call. If the call is currently being executed or - finished running and cannot be cancelled then the method will return - ``False``, otherwise the call will be cancelled and the method will - return ``True``. + Attempt to cancel the call. If the call is currently being executed or + finished running and cannot be cancelled then the method will return + ``False``, otherwise the call will be cancelled and the method will + return ``True``. - .. method:: cancelled() + .. method:: cancelled() - Return ``True`` if the call was successfully cancelled. + Return ``True`` if the call was successfully cancelled. - .. method:: running() + .. method:: running() - Return ``True`` if the call is currently being executed and cannot be - cancelled. + Return ``True`` if the call is currently being executed and cannot be + cancelled. - .. method:: done() + .. method:: done() - Return ``True`` if the call was successfully cancelled or finished - running. + Return ``True`` if the call was successfully cancelled or finished + running. - .. method:: result(timeout=None) + .. method:: result(timeout=None) - Return the value returned by the call. If the call hasn't yet completed - then this method will wait up to *timeout* seconds. If the call hasn't - completed in *timeout* seconds, then a - :exc:`TimeoutError` will be raised. *timeout* can be - an int or float. If *timeout* is not specified or ``None``, there is no - limit to the wait time. + Return the value returned by the call. If the call hasn't yet completed + then this method will wait up to *timeout* seconds. If the call hasn't + completed in *timeout* seconds, then a + :exc:`TimeoutError` will be raised. *timeout* can be + an int or float. If *timeout* is not specified or ``None``, there is no + limit to the wait time. - If the future is cancelled before completing then :exc:`.CancelledError` - will be raised. + If the future is cancelled before completing then :exc:`.CancelledError` + will be raised. - If the call raised an exception, this method will raise the same exception. + If the call raised an exception, this method will raise the same exception. - .. method:: exception(timeout=None) + .. method:: exception(timeout=None) - Return the exception raised by the call. If the call hasn't yet - completed then this method will wait up to *timeout* seconds. If the - call hasn't completed in *timeout* seconds, then a - :exc:`TimeoutError` will be raised. *timeout* can be - an int or float. If *timeout* is not specified or ``None``, there is no - limit to the wait time. + Return the exception raised by the call. If the call hasn't yet + completed then this method will wait up to *timeout* seconds. If the + call hasn't completed in *timeout* seconds, then a + :exc:`TimeoutError` will be raised. *timeout* can be + an int or float. If *timeout* is not specified or ``None``, there is no + limit to the wait time. - If the future is cancelled before completing then :exc:`.CancelledError` - will be raised. + If the future is cancelled before completing then :exc:`.CancelledError` + will be raised. - If the call completed without raising, ``None`` is returned. + If the call completed without raising, ``None`` is returned. - .. method:: add_done_callback(fn) + .. method:: add_done_callback(fn) - Attaches the callable *fn* to the future. *fn* will be called, with the - future as its only argument, when the future is cancelled or finishes - running. + Attaches the callable *fn* to the future. *fn* will be called, with the + future as its only argument, when the future is cancelled or finishes + running. - Added callables are called in the order that they were added and are - always called in a thread belonging to the process that added them. If - the callable raises an :exc:`Exception` subclass, it will be logged and - ignored. If the callable raises a :exc:`BaseException` subclass, the - behavior is undefined. + Added callables are called in the order that they were added and are + always called in a thread belonging to the process that added them. If + the callable raises an :exc:`Exception` subclass, it will be logged and + ignored. If the callable raises a :exc:`BaseException` subclass, the + behavior is undefined. - If the future has already completed or been cancelled, *fn* will be - called immediately. + If the future has already completed or been cancelled, *fn* will be + called immediately. The following :class:`Future` methods are meant for use in unit tests and :class:`Executor` implementations. - .. method:: set_running_or_notify_cancel() + .. method:: set_running_or_notify_cancel() - This method should only be called by :class:`Executor` implementations - before executing the work associated with the :class:`Future` and by unit - tests. + This method should only be called by :class:`Executor` implementations + before executing the work associated with the :class:`Future` and by unit + tests. - If the method returns ``False`` then the :class:`Future` was cancelled, - i.e. :meth:`Future.cancel` was called and returned ``True``. Any threads - waiting on the :class:`Future` completing (i.e. through - :func:`as_completed` or :func:`wait`) will be woken up. + If the method returns ``False`` then the :class:`Future` was cancelled, + i.e. :meth:`Future.cancel` was called and returned ``True``. Any threads + waiting on the :class:`Future` completing (i.e. through + :func:`as_completed` or :func:`wait`) will be woken up. - If the method returns ``True`` then the :class:`Future` was not cancelled - and has been put in the running state, i.e. calls to - :meth:`Future.running` will return ``True``. + If the method returns ``True`` then the :class:`Future` was not cancelled + and has been put in the running state, i.e. calls to + :meth:`Future.running` will return ``True``. - This method can only be called once and cannot be called after - :meth:`Future.set_result` or :meth:`Future.set_exception` have been - called. + This method can only be called once and cannot be called after + :meth:`Future.set_result` or :meth:`Future.set_exception` have been + called. - .. method:: set_result(result) + .. method:: set_result(result) - Sets the result of the work associated with the :class:`Future` to - *result*. + Sets the result of the work associated with the :class:`Future` to + *result*. - This method should only be used by :class:`Executor` implementations and - unit tests. + This method should only be used by :class:`Executor` implementations and + unit tests. - .. versionchanged:: 3.8 - This method raises - :exc:`concurrent.futures.InvalidStateError` if the :class:`Future` is - already done. + .. versionchanged:: 3.8 + This method raises + :exc:`concurrent.futures.InvalidStateError` if the :class:`Future` is + already done. - .. method:: set_exception(exception) + .. method:: set_exception(exception) - Sets the result of the work associated with the :class:`Future` to the - :class:`Exception` *exception*. + Sets the result of the work associated with the :class:`Future` to the + :class:`Exception` *exception*. - This method should only be used by :class:`Executor` implementations and - unit tests. + This method should only be used by :class:`Executor` implementations and + unit tests. - .. versionchanged:: 3.8 - This method raises - :exc:`concurrent.futures.InvalidStateError` if the :class:`Future` is - already done. + .. versionchanged:: 3.8 + This method raises + :exc:`concurrent.futures.InvalidStateError` if the :class:`Future` is + already done. Module Functions ---------------- diff --git a/Doc/library/concurrent.rst b/Doc/library/concurrent.rst index 2eba5365125805..8caea78bbb57e8 100644 --- a/Doc/library/concurrent.rst +++ b/Doc/library/concurrent.rst @@ -1,5 +1,5 @@ -The :mod:`concurrent` package -============================= +The :mod:`!concurrent` package +============================== Currently, there is only one module in this package: diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index a7f75fd6e84f4c..bb282428c5fffc 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -935,8 +935,10 @@ ConfigParser Objects When *default_section* is given, it specifies the name for the special section holding default values for other sections and interpolation purposes - (normally named ``"DEFAULT"``). This value can be retrieved and changed on - runtime using the ``default_section`` instance attribute. + (normally named ``"DEFAULT"``). This value can be retrieved and changed at + runtime using the ``default_section`` instance attribute. This won't + re-evaluate an already parsed config file, but will be used when writing + parsed settings to a new config file. Interpolation behaviour may be customized by providing a custom handler through the *interpolation* argument. ``None`` can be used to turn off diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 38dd552a0363ac..401dc9a320c5e0 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,16 +22,16 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`NoneType` type. + ``None`` is the sole instance of the :data:`~types.NoneType` type. .. data:: NotImplemented A special value which should be returned by the binary special methods - (e.g. :meth:`__eq__`, :meth:`__lt__`, :meth:`__add__`, :meth:`__rsub__`, + (e.g. :meth:`~object.__eq__`, :meth:`~object.__lt__`, :meth:`~object.__add__`, :meth:`~object.__rsub__`, etc.) to indicate that the operation is not implemented with respect to the other type; may be returned by the in-place binary special methods - (e.g. :meth:`__imul__`, :meth:`__iand__`, etc.) for the same purpose. + (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. ``NotImplemented`` is the sole instance of the :data:`types.NotImplementedType` type. diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 7cd081d1f54f43..aab319cbe7405e 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -45,7 +45,7 @@ Functions and classes provided: This function is a :term:`decorator` that can be used to define a factory function for :keyword:`with` statement context managers, without needing to - create a class or separate :meth:`__enter__` and :meth:`__exit__` methods. + create a class or separate :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. While many objects natively support use in with statements, sometimes a resource needs to be managed that isn't a context manager in its own right, @@ -106,8 +106,8 @@ Functions and classes provided: This function is a :term:`decorator` that can be used to define a factory function for :keyword:`async with` statement asynchronous context managers, - without needing to create a class or separate :meth:`__aenter__` and - :meth:`__aexit__` methods. It must be applied to an :term:`asynchronous + without needing to create a class or separate :meth:`~object.__aenter__` and + :meth:`~object.__aexit__` methods. It must be applied to an :term:`asynchronous generator` function. A simple example:: @@ -304,15 +304,15 @@ Functions and classes provided: This context manager is :ref:`reentrant `. - If the code within the :keyword:`!with` block raises an - :exc:`ExceptionGroup`, suppressed exceptions are removed from the + If the code within the :keyword:`!with` block raises a + :exc:`BaseExceptionGroup`, suppressed exceptions are removed from the group. If any exceptions in the group are not suppressed, a group containing them is re-raised. .. versionadded:: 3.4 .. versionchanged:: 3.12 ``suppress`` now supports suppressing exceptions raised as - part of an :exc:`ExceptionGroup`. + part of an :exc:`BaseExceptionGroup`. .. function:: redirect_stdout(new_target) @@ -515,7 +515,7 @@ Functions and classes provided: # the with statement, even if attempts to open files later # in the list raise an exception - The :meth:`__enter__` method returns the :class:`ExitStack` instance, and + The :meth:`~object.__enter__` method returns the :class:`ExitStack` instance, and performs no additional operations. Each instance maintains a stack of registered callbacks that are called in @@ -543,9 +543,9 @@ Functions and classes provided: .. method:: enter_context(cm) - Enters a new context manager and adds its :meth:`__exit__` method to + Enters a new context manager and adds its :meth:`~object.__exit__` method to the callback stack. The return value is the result of the context - manager's own :meth:`__enter__` method. + manager's own :meth:`~object.__enter__` method. These context managers may suppress exceptions just as they normally would if used directly as part of a :keyword:`with` statement. @@ -556,18 +556,18 @@ Functions and classes provided: .. method:: push(exit) - Adds a context manager's :meth:`__exit__` method to the callback stack. + Adds a context manager's :meth:`~object.__exit__` method to the callback stack. As ``__enter__`` is *not* invoked, this method can be used to cover - part of an :meth:`__enter__` implementation with a context manager's own - :meth:`__exit__` method. + part of an :meth:`~object.__enter__` implementation with a context manager's own + :meth:`~object.__exit__` method. If passed an object that is not a context manager, this method assumes it is a callback with the same signature as a context manager's - :meth:`__exit__` method and adds it directly to the callback stack. + :meth:`~object.__exit__` method and adds it directly to the callback stack. By returning true values, these callbacks can suppress exceptions the - same way context manager :meth:`__exit__` methods can. + same way context manager :meth:`~object.__exit__` methods can. The passed in object is returned from the function, allowing this method to be used as a function decorator. @@ -616,12 +616,12 @@ Functions and classes provided: asynchronous context managers, as well as having coroutines for cleanup logic. - The :meth:`close` method is not implemented, :meth:`aclose` must be used + The :meth:`~ExitStack.close` method is not implemented; :meth:`aclose` must be used instead. .. coroutinemethod:: enter_async_context(cm) - Similar to :meth:`enter_context` but expects an asynchronous context + Similar to :meth:`ExitStack.enter_context` but expects an asynchronous context manager. .. versionchanged:: 3.11 @@ -630,16 +630,16 @@ Functions and classes provided: .. method:: push_async_exit(exit) - Similar to :meth:`push` but expects either an asynchronous context manager + Similar to :meth:`ExitStack.push` but expects either an asynchronous context manager or a coroutine function. .. method:: push_async_callback(callback, /, *args, **kwds) - Similar to :meth:`callback` but expects a coroutine function. + Similar to :meth:`ExitStack.callback` but expects a coroutine function. .. coroutinemethod:: aclose() - Similar to :meth:`close` but properly handles awaitables. + Similar to :meth:`ExitStack.close` but properly handles awaitables. Continuing the example for :func:`asynccontextmanager`:: @@ -714,7 +714,7 @@ Cleaning up in an ``__enter__`` implementation As noted in the documentation of :meth:`ExitStack.push`, this method can be useful in cleaning up an already allocated resource if later -steps in the :meth:`__enter__` implementation fail. +steps in the :meth:`~object.__enter__` implementation fail. Here's an example of doing this for a context manager that accepts resource acquisition and release functions, along with an optional validation function, @@ -871,7 +871,7 @@ And also as a function decorator:: Note that there is one additional limitation when using context managers as function decorators: there's no way to access the return value of -:meth:`__enter__`. If that value is needed, then it is still necessary to use +:meth:`~object.__enter__`. If that value is needed, then it is still necessary to use an explicit ``with`` statement. .. seealso:: diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 8f32477ed508c3..74333b2e934814 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -17,14 +17,22 @@ operations (explained below). Interface summary: -.. function:: copy(x) +.. function:: copy(obj) - Return a shallow copy of *x*. + Return a shallow copy of *obj*. -.. function:: deepcopy(x[, memo]) +.. function:: deepcopy(obj[, memo]) - Return a deep copy of *x*. + Return a deep copy of *obj*. + + +.. function:: replace(obj, /, **changes) + + Creates a new object of the same type as *obj*, replacing fields with values + from *changes*. + + .. versionadded:: 3.13 .. exception:: Error @@ -79,14 +87,40 @@ pickle functions from the :mod:`copyreg` module. single: __copy__() (copy protocol) single: __deepcopy__() (copy protocol) +.. currentmodule:: None + In order for a class to define its own copy implementation, it can define -special methods :meth:`__copy__` and :meth:`__deepcopy__`. The former is called -to implement the shallow copy operation; no additional arguments are passed. -The latter is called to implement the deep copy operation; it is passed one -argument, the ``memo`` dictionary. If the :meth:`__deepcopy__` implementation needs -to make a deep copy of a component, it should call the :func:`deepcopy` function -with the component as first argument and the memo dictionary as second argument. -The memo dictionary should be treated as an opaque object. +special methods :meth:`~object.__copy__` and :meth:`~object.__deepcopy__`. + +.. method:: object.__copy__(self) + :noindexentry: + + Called to implement the shallow copy operation; + no additional arguments are passed. + +.. method:: object.__deepcopy__(self, memo) + :noindexentry: + + Called to implement the deep copy operation; it is passed one + argument, the *memo* dictionary. If the ``__deepcopy__`` implementation needs + to make a deep copy of a component, it should call the :func:`~copy.deepcopy` function + with the component as first argument and the *memo* dictionary as second argument. + The *memo* dictionary should be treated as an opaque object. + + +.. index:: + single: __replace__() (replace protocol) + +Function :func:`!copy.replace` is more limited +than :func:`~copy.copy` and :func:`~copy.deepcopy`, +and only supports named tuples created by :func:`~collections.namedtuple`, +:mod:`dataclasses`, and other classes which define method :meth:`~object.__replace__`. + +.. method:: object.__replace__(self, /, **changes) + :noindexentry: + + This method should create a new object of the same type, + replacing fields with values from *changes*. .. seealso:: diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst deleted file mode 100644 index 740084b40c5ac9..00000000000000 --- a/Doc/library/crypt.rst +++ /dev/null @@ -1,182 +0,0 @@ -:mod:`crypt` --- Function to check Unix passwords -================================================= - -.. module:: crypt - :platform: Unix - :synopsis: The crypt() function used to check Unix passwords. - :deprecated: - -.. moduleauthor:: Steven D. Majewski -.. sectionauthor:: Steven D. Majewski -.. sectionauthor:: Peter Funk - -**Source code:** :source:`Lib/crypt.py` - -.. index:: - single: crypt(3) - pair: cipher; DES - -.. deprecated-removed:: 3.11 3.13 - The :mod:`crypt` module is deprecated - (see :pep:`PEP 594 <594#crypt>` for details and alternatives). - The :mod:`hashlib` module is a potential replacement for certain use cases. - --------------- - -This module implements an interface to the :manpage:`crypt(3)` routine, which is -a one-way hash function based upon a modified DES algorithm; see the Unix man -page for further details. Possible uses include storing hashed passwords -so you can check passwords without storing the actual password, or attempting -to crack Unix passwords with a dictionary. - -.. index:: single: crypt(3) - -Notice that the behavior of this module depends on the actual implementation of -the :manpage:`crypt(3)` routine in the running system. Therefore, any -extensions available on the current implementation will also be available on -this module. - -.. availability:: Unix, not VxWorks. - -.. include:: ../includes/wasm-notavail.rst - -Hashing Methods ---------------- - -.. versionadded:: 3.3 - -The :mod:`crypt` module defines the list of hashing methods (not all methods -are available on all platforms): - -.. data:: METHOD_SHA512 - - A Modular Crypt Format method with 16 character salt and 86 character - hash based on the SHA-512 hash function. This is the strongest method. - -.. data:: METHOD_SHA256 - - Another Modular Crypt Format method with 16 character salt and 43 - character hash based on the SHA-256 hash function. - -.. data:: METHOD_BLOWFISH - - Another Modular Crypt Format method with 22 character salt and 31 - character hash based on the Blowfish cipher. - - .. versionadded:: 3.7 - -.. data:: METHOD_MD5 - - Another Modular Crypt Format method with 8 character salt and 22 - character hash based on the MD5 hash function. - -.. data:: METHOD_CRYPT - - The traditional method with a 2 character salt and 13 characters of - hash. This is the weakest method. - - -Module Attributes ------------------ - -.. versionadded:: 3.3 - -.. attribute:: methods - - A list of available password hashing algorithms, as - ``crypt.METHOD_*`` objects. This list is sorted from strongest to - weakest. - - -Module Functions ----------------- - -The :mod:`crypt` module defines the following functions: - -.. function:: crypt(word, salt=None) - - *word* will usually be a user's password as typed at a prompt or in a graphical - interface. The optional *salt* is either a string as returned from - :func:`mksalt`, one of the ``crypt.METHOD_*`` values (though not all - may be available on all platforms), or a full encrypted password - including salt, as returned by this function. If *salt* is not - provided, the strongest method available in :attr:`methods` will be used. - - Checking a password is usually done by passing the plain-text password - as *word* and the full results of a previous :func:`crypt` call, - which should be the same as the results of this call. - - *salt* (either a random 2 or 16 character string, possibly prefixed with - ``$digit$`` to indicate the method) which will be used to perturb the - encryption algorithm. The characters in *salt* must be in the set - ``[./a-zA-Z0-9]``, with the exception of Modular Crypt Format which - prefixes a ``$digit$``. - - Returns the hashed password as a string, which will be composed of - characters from the same alphabet as the salt. - - .. index:: single: crypt(3) - - Since a few :manpage:`crypt(3)` extensions allow different values, with - different sizes in the *salt*, it is recommended to use the full crypted - password as salt when checking for a password. - - .. versionchanged:: 3.3 - Accept ``crypt.METHOD_*`` values in addition to strings for *salt*. - - -.. function:: mksalt(method=None, *, rounds=None) - - Return a randomly generated salt of the specified method. If no - *method* is given, the strongest method available in :attr:`methods` is - used. - - The return value is a string suitable for passing as the *salt* argument - to :func:`crypt`. - - *rounds* specifies the number of rounds for ``METHOD_SHA256``, - ``METHOD_SHA512`` and ``METHOD_BLOWFISH``. - For ``METHOD_SHA256`` and ``METHOD_SHA512`` it must be an integer between - ``1000`` and ``999_999_999``, the default is ``5000``. For - ``METHOD_BLOWFISH`` it must be a power of two between ``16`` (2\ :sup:`4`) - and ``2_147_483_648`` (2\ :sup:`31`), the default is ``4096`` - (2\ :sup:`12`). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.7 - Added the *rounds* parameter. - - -Examples --------- - -A simple example illustrating typical use (a constant-time comparison -operation is needed to limit exposure to timing attacks. -:func:`hmac.compare_digest` is suitable for this purpose):: - - import pwd - import crypt - import getpass - from hmac import compare_digest as compare_hash - - def login(): - username = input('Python login: ') - cryptedpasswd = pwd.getpwnam(username)[1] - if cryptedpasswd: - if cryptedpasswd == 'x' or cryptedpasswd == '*': - raise ValueError('no support for shadow passwords') - cleartext = getpass.getpass() - return compare_hash(crypt.crypt(cleartext, cryptedpasswd), cryptedpasswd) - else: - return True - -To generate a hash of a password using the strongest available method and -check it against the original:: - - import crypt - from hmac import compare_digest as compare_hash - - hashed = crypt.crypt(plaintext) - if not compare_hash(hashed, crypt.crypt(plaintext, hashed)): - raise ValueError("hashed version doesn't validate against original") diff --git a/Doc/library/crypto.rst b/Doc/library/crypto.rst index ae45549a6d8941..5a3b7a807213f4 100644 --- a/Doc/library/crypto.rst +++ b/Doc/library/crypto.rst @@ -8,7 +8,6 @@ Cryptographic Services The modules described in this chapter implement various algorithms of a cryptographic nature. They are available at the discretion of the installation. -On Unix systems, the :mod:`crypt` module may also be available. Here's an overview: diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 64baa69be4af31..aba398b8ee1e54 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -288,9 +288,9 @@ The :mod:`csv` module defines the following classes: Inspecting each column, one of two key criteria will be considered to estimate if the sample contains a header: - - the second through n-th rows contain numeric values - - the second through n-th rows contain strings where at least one value's - length differs from that of the putative header of that column. + - the second through n-th rows contain numeric values + - the second through n-th rows contain strings where at least one value's + length differs from that of the putative header of that column. Twenty rows after the first row are sampled; if more than half of columns + rows meet the criteria, :const:`True` is returned. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 81509c0920bb6e..ef3a9a0f5898af 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -41,7 +41,7 @@ You load libraries by accessing them as attributes of these objects. *cdll* loads libraries which export functions using the standard ``cdecl`` calling convention, while *windll* libraries call functions using the ``stdcall`` calling convention. *oledll* also uses the ``stdcall`` calling convention, and -assumes the functions return a Windows :c:type:`HRESULT` error code. The error +assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -72,8 +72,9 @@ Windows appends the usual ``.dll`` file suffix automatically. On Linux, it is required to specify the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the -:meth:`LoadLibrary` method of the dll loaders should be used, or you should load -the library by creating an instance of CDLL by calling the constructor:: +:meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, +or you should load the library by creating an instance of CDLL by calling +the constructor:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -220,7 +221,7 @@ Fundamental data types +----------------------+------------------------------------------+----------------------------+ | :class:`c_char` | :c:expr:`char` | 1-character bytes object | +----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:expr:`wchar_t` | 1-character string | +| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | +----------------------+------------------------------------------+----------------------------+ | :class:`c_byte` | :c:expr:`char` | int | +----------------------+------------------------------------------+----------------------------+ @@ -243,9 +244,9 @@ Fundamental data types | :class:`c_ulonglong` | :c:expr:`unsigned __int64` or | int | | | :c:expr:`unsigned long long` | | +----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:expr:`size_t` | int | +| :class:`c_size_t` | :c:type:`size_t` | int | +----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:expr:`ssize_t` or | int | +| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | | | :c:expr:`Py_ssize_t` | | +----------------------+------------------------------------------+----------------------------+ | :class:`c_time_t` | :c:type:`time_t` | int | @@ -333,9 +334,9 @@ property:: 10 b'Hi\x00lo\x00\x00\x00\x00\x00' >>> -The :func:`create_string_buffer` function replaces the old :func:`c_buffer` +The :func:`create_string_buffer` function replaces the old :func:`!c_buffer` function (which is still available as an alias). To create a mutable memory -block containing unicode characters of the C type :c:expr:`wchar_t`, use the +block containing unicode characters of the C type :c:type:`wchar_t`, use the :func:`create_unicode_buffer` function. @@ -361,7 +362,7 @@ from within *IDLE* or *PythonWin*:: >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 + ctypes.ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 >>> As has been mentioned before, all Python types except integers, strings, and @@ -383,15 +384,15 @@ as calling functions with a fixed number of parameters. On some platforms, and i particular ARM64 for Apple Platforms, the calling convention for variadic functions is different than that for regular functions. -On those platforms it is required to specify the *argtypes* attribute for the -regular, non-variadic, function arguments: +On those platforms it is required to specify the :attr:`~_FuncPtr.argtypes` +attribute for the regular, non-variadic, function arguments: .. code-block:: python3 libc.printf.argtypes = [ctypes.c_char_p] Because specifying the attribute does not inhibit portability it is advised to always -specify ``argtypes`` for all variadic functions. +specify :attr:`~_FuncPtr.argtypes` for all variadic functions. .. _ctypes-calling-functions-with-own-custom-data-types: @@ -400,9 +401,10 @@ Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`_as_parameter_` attribute and uses this as the function argument. Of -course, it must be one of integer, string, or bytes:: +your own classes be used as function arguments. :mod:`ctypes` looks for an +:attr:`!_as_parameter_` attribute and uses this as the function argument. The +attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: ... def __init__(self, number): @@ -414,7 +416,7 @@ course, it must be one of integer, string, or bytes:: 19 >>> -If you don't want to store the instance's data in the :attr:`_as_parameter_` +If you don't want to store the instance's data in the :attr:`!_as_parameter_` instance variable, you could define a :class:`property` which makes the attribute available on request. @@ -425,9 +427,9 @@ Specifying the required argument types (function prototypes) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`argtypes` attribute. +DLLs by setting the :attr:`~_FuncPtr.argtypes` attribute. -:attr:`argtypes` must be a sequence of C data types (the ``printf`` function is +:attr:`~_FuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is probably not a good example here, because it takes a variable number and different types of parameters depending on the format string, on the other hand this is quite handy to experiment with this feature):: @@ -444,21 +446,21 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: TypeError: wrong type + ctypes.ArgumentError: argument 2: TypeError: 'int' object cannot be interpreted as ctypes.c_char_p >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 13 >>> If you have defined your own classes which you pass to function calls, you have -to implement a :meth:`from_param` class method for them to be able to use them -in the :attr:`argtypes` sequence. The :meth:`from_param` class method receives +to implement a :meth:`~_CData.from_param` class method for them to be able to use them +in the :attr:`~_FuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the -object itself, its :attr:`_as_parameter_` attribute, or whatever you want to +object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an integer, string, bytes, a :mod:`ctypes` instance, or an object with an -:attr:`_as_parameter_` attribute. +:attr:`!_as_parameter_` attribute. .. _ctypes-return-types: @@ -475,16 +477,16 @@ Return types By default functions are assumed to return the C :c:expr:`int` type. Other -return types can be specified by setting the :attr:`restype` attribute of the +return types can be specified by setting the :attr:`~_FuncPtr.restype` attribute of the function object. -The C prototype of ``time()`` is ``time_t time(time_t *)``. Because ``time_t`` -might be of a different type than the default return type ``int``, you should -specify the ``restype``:: +The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` +might be of a different type than the default return type :c:expr:`int`, you should +specify the :attr:`!restype` attribute:: >>> libc.time.restype = c_time_t -The argument types can be specified using ``argtypes``:: +The argument types can be specified using :attr:`~_FuncPtr.argtypes`:: >>> libc.time.argtypes = (POINTER(c_time_t),) @@ -493,7 +495,7 @@ To call the function with a ``NULL`` pointer as first argument, use ``None``:: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 -Here is a more advanced example, it uses the ``strchr`` function, which expects +Here is a more advanced example, it uses the :func:`!strchr` function, which expects a string pointer and a char, and returns a pointer to a string:: >>> strchr = libc.strchr @@ -506,8 +508,8 @@ a string pointer and a char, and returns a pointer to a string:: None >>> -If you want to avoid the ``ord("x")`` calls above, you can set the -:attr:`argtypes` attribute, and the second argument will be converted from a +If you want to avoid the :func:`ord("x") ` calls above, you can set the +:attr:`~_FuncPtr.argtypes` attribute, and the second argument will be converted from a single character Python bytes object into a C char: .. doctest:: @@ -526,7 +528,7 @@ single character Python bytes object into a C char: >>> You can also use a callable Python object (a function or a class for example) as -the :attr:`restype` attribute, if the foreign function returns an integer. The +the :attr:`~_FuncPtr.restype` attribute, if the foreign function returns an integer. The callable will be called with the *integer* the C function returns, and the result of this call will be used as the result of your function call. This is useful to check for error return values and automatically raise an exception:: @@ -554,7 +556,8 @@ get the string representation of an error code, and *returns* an exception. :func:`GetLastError` to retrieve it. Please note that a much more powerful error checking mechanism is available -through the :attr:`errcheck` attribute; see the reference manual for details. +through the :attr:`~_FuncPtr.errcheck` attribute; +see the reference manual for details. .. _ctypes-passing-pointers: @@ -592,7 +595,7 @@ Structures and unions Structures and unions must derive from the :class:`Structure` and :class:`Union` base classes which are defined in the :mod:`ctypes` module. Each subclass must -define a :attr:`_fields_` attribute. :attr:`_fields_` must be a list of +define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other @@ -664,9 +667,9 @@ Structure/union alignment and byte order By default, Structure and Union fields are aligned in the same way the C compiler does it. It is possible to override this behavior by specifying a -:attr:`_pack_` class attribute in the subclass definition. This must be set to a -positive integer and specifies the maximum alignment for the fields. This is -what ``#pragma pack(n)`` also does in MSVC. +:attr:`~Structure._pack_` class attribute in the subclass definition. +This must be set to a positive integer and specifies the maximum alignment for the fields. +This is what ``#pragma pack(n)`` also does in MSVC. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -682,7 +685,7 @@ Bit fields in structures and unions It is possible to create structures and unions containing bit fields. Bit fields are only possible for integer fields, the bit width is specified as the third -item in the :attr:`_fields_` tuples:: +item in the :attr:`~Structure._fields_` tuples:: >>> class Int(Structure): ... _fields_ = [("first_16", c_int, 16), @@ -853,7 +856,7 @@ Type conversions ^^^^^^^^^^^^^^^^ Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`argtypes` list of a function or as the type of +``POINTER(c_int)`` in the :attr:`~_FuncPtr.argtypes` list of a function or as the type of a member field in a structure definition, only instances of exactly the same type are accepted. There are some exceptions to this rule, where ctypes accepts other objects. For example, you can pass compatible array instances instead of @@ -874,7 +877,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`~_FuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -950,8 +953,8 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the :attr:`_fields_` -attribute later, after the class statement:: +In :mod:`ctypes`, we can define the ``cell`` class and set the +:attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * >>> class cell(Structure): @@ -1001,8 +1004,8 @@ argument, and the callback functions expected argument types as the remaining arguments. I will present an example here which uses the standard C library's -:c:func:`qsort` function, that is used to sort items with the help of a callback -function. :c:func:`qsort` will be used to sort an array of integers:: +:c:func:`!qsort` function, that is used to sort items with the help of a callback +function. :c:func:`!qsort` will be used to sort an array of integers:: >>> IntArray5 = c_int * 5 >>> ia = IntArray5(5, 1, 7, 33, 99) @@ -1010,7 +1013,7 @@ function. :c:func:`qsort` will be used to sort an array of integers:: >>> qsort.restype = None >>> -:func:`qsort` must be called with a pointer to the data to sort, the number of +:func:`!qsort` must be called with a pointer to the data to sort, the number of items in the data array, the size of one item, and a pointer to the comparison function, the callback. The callback will then be called with two pointers to items, and it must return a negative integer if the first item is smaller than @@ -1102,7 +1105,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`in_dll` class methods of +:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1292,13 +1295,13 @@ Finding shared libraries When programming in a compiled language, shared libraries are accessed when compiling/linking a program, and when the program is run. -The purpose of the :func:`find_library` function is to locate a library in a way +The purpose of the :func:`~ctypes.util.find_library` function is to locate a library in a way similar to what the compiler or runtime loader does (on platforms with several versions of a shared library the most recent should be loaded), while the ctypes library loaders act like when a program is run, and call the runtime loader directly. -The :mod:`ctypes.util` module provides a function which can help to determine +The :mod:`!ctypes.util` module provides a function which can help to determine the library to load. @@ -1313,7 +1316,7 @@ the library to load. The exact functionality is system dependent. -On Linux, :func:`find_library` tries to run external programs +On Linux, :func:`~ctypes.util.find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. It returns the filename of the library file. @@ -1332,7 +1335,7 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS, :func:`find_library` tries several predefined naming schemes and paths +On macOS, :func:`~ctypes.util.find_library` tries several predefined naming schemes and paths to locate the library, and returns a full pathname if successful:: >>> from ctypes.util import find_library @@ -1346,13 +1349,13 @@ to locate the library, and returns a full pathname if successful:: '/System/Library/Frameworks/AGL.framework/AGL' >>> -On Windows, :func:`find_library` searches along the system search path, and +On Windows, :func:`~ctypes.util.find_library` searches along the system search path, and returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`find_library` to locate the library at runtime. +module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. .. _ctypes-loading-shared-libraries: @@ -1400,7 +1403,8 @@ way is to instantiate one of the following classes: failure, an :class:`OSError` is automatically raised. .. versionchanged:: 3.3 - :exc:`WindowsError` used to be raised. + :exc:`WindowsError` used to be raised, + which is now an alias of :exc:`OSError`. .. versionchanged:: 3.12 @@ -1437,9 +1441,9 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms ``dlopen`` or ``LoadLibrary`` -function is used to load the library into the process, and to get a handle to -it. +parameter, otherwise the underlying platforms :c:func:`!dlopen` or +:c:func:`!LoadLibrary` function is used to load the library into +the process, and to get a handle to it. The *mode* parameter can be used to specify how the library is loaded. For details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is @@ -1459,7 +1463,7 @@ to a new value and returns the former value. The *use_last_error* parameter, when set to true, enables the same mechanism for the Windows error code which is managed by the :func:`GetLastError` and -:func:`SetLastError` Windows API functions; :func:`ctypes.get_last_error` and +:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and :func:`ctypes.set_last_error` are used to request and change the ctypes private copy of the windows error code. @@ -1522,8 +1526,8 @@ underscore to not clash with exported function names: Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the -:meth:`LoadLibrary` method, or by retrieving the library as attribute of the -loader instance. +:meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as +attribute of the loader instance. .. class:: LibraryLoader(dlltype) @@ -1531,7 +1535,7 @@ loader instance. Class which loads shared libraries. *dlltype* should be one of the :class:`CDLL`, :class:`PyDLL`, :class:`WinDLL`, or :class:`OleDLL` types. - :meth:`__getattr__` has special behavior: It allows loading a shared library by + :meth:`!__getattr__` has special behavior: It allows loading a shared library by accessing it as attribute of a library loader instance. The result is cached, so repeated attribute accesses return the same library each time. @@ -1576,7 +1580,7 @@ object is available: An instance of :class:`PyDLL` that exposes Python C API functions as attributes. Note that all these functions are assumed to return C :c:expr:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`restype` attribute to use these functions. + the correct :attr:`!restype` attribute to use these functions. .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader @@ -1628,7 +1632,7 @@ They are instances of a private class: the callable will be called with this integer, allowing further processing or error checking. Using this is deprecated, for more flexible post processing or error checking use a ctypes data type as - :attr:`restype` and assign a callable to the :attr:`errcheck` attribute. + :attr:`!restype` and assign a callable to the :attr:`errcheck` attribute. .. attribute:: argtypes @@ -1639,14 +1643,14 @@ They are instances of a private class: unspecified arguments as well. When a foreign function is called, each actual argument is passed to the - :meth:`from_param` class method of the items in the :attr:`argtypes` + :meth:`~_CData.from_param` class method of the items in the :attr:`argtypes` tuple, this method allows adapting the actual argument to an object that the foreign function accepts. For example, a :class:`c_char_p` item in the :attr:`argtypes` tuple will convert a string passed as argument into a bytes object using ctypes conversion rules. New: It is now possible to put items in argtypes which are not ctypes - types, but each item must have a :meth:`from_param` method which returns a + types, but each item must have a :meth:`~_CData.from_param` method which returns a value usable as argument (integer, string, ctypes instance). This allows defining adapters that can adapt custom objects as function parameters. @@ -1660,7 +1664,7 @@ They are instances of a private class: :module: *result* is what the foreign function returns, as specified by the - :attr:`restype` attribute. + :attr:`!restype` attribute. *func* is the foreign function object itself, this allows reusing the same callable object to check or post process the results of several @@ -1734,70 +1738,70 @@ See :ref:`ctypes-callback-functions` for examples. Function prototypes created by these factory functions can be instantiated in different ways, depending on the type and number of the parameters in the call: +.. function:: prototype(address) + :noindex: + :module: - .. function:: prototype(address) - :noindex: - :module: + Returns a foreign function at the specified address which must be an integer. - Returns a foreign function at the specified address which must be an integer. +.. function:: prototype(callable) + :noindex: + :module: - .. function:: prototype(callable) - :noindex: - :module: + Create a C callable function (a callback function) from a Python *callable*. - Create a C callable function (a callback function) from a Python *callable*. +.. function:: prototype(func_spec[, paramflags]) + :noindex: + :module: - .. function:: prototype(func_spec[, paramflags]) - :noindex: - :module: + Returns a foreign function exported by a shared library. *func_spec* must + be a 2-tuple ``(name_or_ordinal, library)``. The first item is the name of + the exported function as string, or the ordinal of the exported function + as small integer. The second item is the shared library instance. - Returns a foreign function exported by a shared library. *func_spec* must - be a 2-tuple ``(name_or_ordinal, library)``. The first item is the name of - the exported function as string, or the ordinal of the exported function - as small integer. The second item is the shared library instance. +.. function:: prototype(vtbl_index, name[, paramflags[, iid]]) + :noindex: + :module: - .. function:: prototype(vtbl_index, name[, paramflags[, iid]]) - :noindex: - :module: + Returns a foreign function that will call a COM method. *vtbl_index* is + the index into the virtual function table, a small non-negative + integer. *name* is name of the COM method. *iid* is an optional pointer to + the interface identifier which is used in extended error reporting. - Returns a foreign function that will call a COM method. *vtbl_index* is - the index into the virtual function table, a small non-negative - integer. *name* is name of the COM method. *iid* is an optional pointer to - the interface identifier which is used in extended error reporting. + COM methods use a special calling convention: They require a pointer to + the COM interface as first argument, in addition to those parameters that + are specified in the :attr:`!argtypes` tuple. - COM methods use a special calling convention: They require a pointer to - the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`argtypes` tuple. +The optional *paramflags* parameter creates foreign function wrappers with much +more functionality than the features described above. - The optional *paramflags* parameter creates foreign function wrappers with much - more functionality than the features described above. +*paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`. - *paramflags* must be a tuple of the same length as :attr:`argtypes`. +Each item in this tuple contains further information about a parameter, it must +be a tuple containing one, two, or three items. - Each item in this tuple contains further information about a parameter, it must - be a tuple containing one, two, or three items. +The first item is an integer containing a combination of direction +flags for the parameter: - The first item is an integer containing a combination of direction - flags for the parameter: + 1 + Specifies an input parameter to the function. - 1 - Specifies an input parameter to the function. + 2 + Output parameter. The foreign function fills in a value. - 2 - Output parameter. The foreign function fills in a value. + 4 + Input parameter which defaults to the integer zero. - 4 - Input parameter which defaults to the integer zero. +The optional second item is the parameter name as string. If this is specified, +the foreign function can be called with named parameters. - The optional second item is the parameter name as string. If this is specified, - the foreign function can be called with named parameters. +The optional third item is the default value for this parameter. - The optional third item is the default value for this parameter. -This example demonstrates how to wrap the Windows ``MessageBoxW`` function so +The following example demonstrates how to wrap the Windows ``MessageBoxW`` function so that it supports default parameters and named arguments. The C declaration from the windows header file is this:: @@ -1845,7 +1849,7 @@ value if there is a single one, or a tuple containing the output parameter values when there are more than one, so the GetWindowRect function now returns a RECT instance, when called. -Output parameters can be combined with the :attr:`errcheck` protocol to do +Output parameters can be combined with the :attr:`~_FuncPtr.errcheck` protocol to do further output processing and error checking. The win32 ``GetWindowRect`` api function returns a ``BOOL`` to signal success or failure, so this function could do the error checking, and raises an exception when the api call failed:: @@ -1858,7 +1862,7 @@ do the error checking, and raises an exception when the api call failed:: >>> GetWindowRect.errcheck = errcheck >>> -If the :attr:`errcheck` function returns the argument tuple it receives +If the :attr:`~_FuncPtr.errcheck` function returns the argument tuple it receives unchanged, :mod:`ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them @@ -2008,7 +2012,7 @@ Utility functions .. function:: get_last_error() Windows only: returns the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread. + :data:`!LastError` variable in the calling thread. .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error @@ -2026,17 +2030,17 @@ Utility functions specifying an address, or a ctypes instance. -.. function:: POINTER(type) +.. function:: POINTER(type, /) - This factory function creates and returns a new ctypes pointer type. Pointer - types are cached and reused internally, so calling this function repeatedly is - cheap. *type* must be a ctypes type. + Create and return a new ctypes pointer type. Pointer types are cached and + reused internally, so calling this function repeatedly is cheap. + *type* must be a ctypes type. -.. function:: pointer(obj) +.. function:: pointer(obj, /) - This function creates a new pointer instance, pointing to *obj*. The returned - object is of the type ``POINTER(type(obj))``. + Create a new pointer instance, pointing to *obj*. + The returned object is of the type ``POINTER(type(obj))``. Note: If you just want to pass a pointer to an object to a foreign function call, you should use ``byref(obj)`` which is much faster. @@ -2061,7 +2065,7 @@ Utility functions .. function:: set_last_error(value) Windows only: set the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread to *value* and return the + :data:`!LastError` variable in the calling thread to *value* and return the previous value. .. audit-event:: ctypes.set_last_error error ctypes.set_last_error @@ -2085,13 +2089,14 @@ Utility functions .. function:: WinError(code=None, descr=None) Windows only: this function is probably the worst-named thing in ctypes. It - creates an instance of OSError. If *code* is not specified, + creates an instance of :exc:`OSError`. If *code* is not specified, ``GetLastError`` is called to determine the error code. If *descr* is not specified, :func:`FormatError` is called to get a textual description of the error. .. versionchanged:: 3.3 - An instance of :exc:`WindowsError` used to be created. + An instance of :exc:`WindowsError` used to be created, which is now an + alias of :exc:`OSError`. .. function:: wstring_at(address, size=-1) @@ -2157,8 +2162,8 @@ Data types This method adapts *obj* to a ctypes type. It is called with the actual object used in a foreign function call when the type is present in the - foreign function's :attr:`argtypes` tuple; it must return an object that - can be used as a function call parameter. + foreign function's :attr:`~_FuncPtr.argtypes` tuple; + it must return an object that can be used as a function call parameter. All ctypes data types have a default implementation of this classmethod that normally returns *obj* if that is an instance of the type. Some @@ -2223,13 +2228,13 @@ Fundamental data types Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently converted to native Python types. In other words, if a foreign function has a -:attr:`restype` of :class:`c_char_p`, you will always receive a Python bytes +:attr:`~_FuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes object, *not* a :class:`c_char_p` instance. .. XXX above is false, it actually returns a Unicode string Subclasses of fundamental data types do *not* inherit this behavior. So, if a -foreign functions :attr:`restype` is a subclass of :class:`c_void_p`, you will +foreign functions :attr:`!restype` is a subclass of :class:`c_void_p`, you will receive an instance of this subclass from the function call. Of course, you can get the value of the pointer by accessing the ``value`` attribute. @@ -2407,7 +2412,7 @@ These are the fundamental ctypes data types: .. class:: c_wchar - Represents the C :c:expr:`wchar_t` datatype, and interprets the value as a + Represents the C :c:type:`wchar_t` datatype, and interprets the value as a single character unicode string. The constructor accepts an optional string initializer, the length of the string must be exactly one character. @@ -2428,7 +2433,7 @@ These are the fundamental ctypes data types: .. class:: HRESULT - Windows only: Represents a :c:type:`HRESULT` value, which contains success or + Windows only: Represents a :c:type:`!HRESULT` value, which contains success or error information for a function or method call. @@ -2437,9 +2442,9 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`PyObject *` datatype. Calling this without an argument creates a ``NULL`` :c:expr:`PyObject *` pointer. -The :mod:`ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`HWND`, :c:type:`WPARAM`, or :c:type:`DWORD`. Some -useful structures like :c:type:`MSG` or :c:type:`RECT` are also defined. +The :mod:`!ctypes.wintypes` module provides quite some other Windows specific +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. .. _ctypes-structured-data-types: diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index cf208f3ba0db36..9b8a98f05f7cbb 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -641,7 +641,8 @@ The module :mod:`curses` defines the following functions: .. function:: update_lines_cols() - Update :envvar:`LINES` and :envvar:`COLS`. Useful for detecting manual screen resize. + Update the :const:`LINES` and :const:`COLS` module variables. + Useful for detecting manual screen resize. .. versionadded:: 3.5 @@ -1342,10 +1343,27 @@ The :mod:`curses` module defines the following data members: .. data:: COLORS The maximum number of colors the terminal can support. + It is defined only after the call to :func:`start_color`. .. data:: COLOR_PAIRS The maximum number of color pairs the terminal can support. + It is defined only after the call to :func:`start_color`. + +.. data:: COLS + + The width of the screen, i.e., the number of columns. + It is defined only after the call to :func:`initscr`. + Updated by :func:`update_lines_cols`, :func:`resizeterm` and + :func:`resize_term`. + +.. data:: LINES + + The height of the screen, i.e., the number of lines. + It is defined only after the call to :func:`initscr`. + Updated by :func:`update_lines_cols`, :func:`resizeterm` and + :func:`resize_term`. + Some constants are available to specify character cell attributes. The exact constants available are system dependent. @@ -1630,6 +1648,8 @@ keys); also, the following keypad mappings are standard: | :kbd:`Page Down` | KEY_NPAGE | +------------------+-----------+ +.. _curses-acs-codes: + The following table lists characters from the alternate character set. These are inherited from the VT100 terminal, and will generally be available on software emulations such as X terminals. When there is no graphic available, curses @@ -1751,9 +1771,9 @@ The following table lists mouse button constants used by :meth:`getmouse`: | .. data:: BUTTON_ALT | Control was down during button state change | +----------------------------------+---------------------------------------------+ - .. versionchanged:: 3.10 - The ``BUTTON5_*`` constants are now exposed if they are provided by the - underlying curses library. +.. versionchanged:: 3.10 + The ``BUTTON5_*`` constants are now exposed if they are provided by the + underlying curses library. The following table lists the predefined colors: diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index a5b20149921042..bbbbcb00d8fef8 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -319,13 +319,11 @@ Module contents module-level method (see below). Users should never instantiate a :class:`Field` object directly. Its documented attributes are: - - ``name``: The name of the field. - - - ``type``: The type of the field. - - - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, ``metadata``, and ``kw_only`` have the identical - meaning and values as they do in the :func:`field` function. + - ``name``: The name of the field. + - ``type``: The type of the field. + - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, + ``compare``, ``metadata``, and ``kw_only`` have the identical + meaning and values as they do in the :func:`field` function. Other attributes may exist, but they are private and must not be inspected or relied on. @@ -456,6 +454,8 @@ Module contents ``replace()`` (or similarly named) method which handles instance copying. + Dataclass instances are also supported by generic function :func:`copy.replace`. + .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass or an instance of one, @@ -609,7 +609,7 @@ methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and -must use :meth:`~object.__setattr__`. +must use :meth:`!object.__setattr__`. Inheritance ----------- @@ -738,7 +738,7 @@ for ``x`` when creating a class instance will share the same copy of ``x``. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`dataclass` decorator will raise a :exc:`TypeError` if it +:func:`dataclass` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index bed19ad145a20c..1bab1fb9bd9464 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -19,6 +19,10 @@ The :mod:`datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. +.. tip:: + + Skip to :ref:`the format codes `. + .. seealso:: Module :mod:`calendar` @@ -33,6 +37,11 @@ on efficient attribute extraction for output formatting and manipulation. Package `dateutil `_ Third-party library with expanded time zone and parsing support. + Package `DateType `_ + Third-party library that introduces distinct static types to e.g. allow + :term:`static type checkers ` + to differentiate between naive and aware datetimes. + .. _datetime-naive-aware: Aware and Naive Objects @@ -644,6 +653,9 @@ Instance methods: >>> d.replace(day=26) datetime.date(2002, 12, 26) + :class:`date` objects are also supported by generic function + :func:`copy.replace`. + .. method:: date.timetuple() @@ -1243,6 +1255,9 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive datetime from an aware datetime with no conversion of date and time data. + :class:`datetime` objects are also supported by generic function + :func:`copy.replace`. + .. versionadded:: 3.6 Added the ``fold`` argument. @@ -1819,6 +1834,9 @@ Instance methods: ``tzinfo=None`` can be specified to create a naive :class:`.time` from an aware :class:`.time`, without conversion of the time data. + :class:`time` objects are also supported by generic function + :func:`copy.replace`. + .. versionadded:: 3.6 Added the ``fold`` argument. @@ -2322,6 +2340,8 @@ versus :meth:`strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ + .. _format-codes: + :meth:`strftime` and :meth:`strptime` Format Codes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 2be499337a2a15..766847b971b645 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -245,6 +245,13 @@ supported. Close the ``gdbm`` database. + .. method:: gdbm.clear() + + Remove all items from the ``gdbm`` database. + + .. versionadded:: 3.13 + + :mod:`dbm.ndbm` --- Interface based on ndbm ------------------------------------------- @@ -313,6 +320,12 @@ to locate the appropriate header file to simplify building this module. Close the ``ndbm`` database. + .. method:: ndbm.clear() + + Remove all items from the ``ndbm`` database. + + .. versionadded:: 3.13 + :mod:`dbm.dumb` --- Portable DBM implementation ----------------------------------------------- diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 0b4a4973cb4da0..fb8bbf72adf268 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -743,12 +743,23 @@ Decimal objects .. method:: normalize(context=None) - Normalize the number by stripping the rightmost trailing zeros and - converting any result equal to ``Decimal('0')`` to - ``Decimal('0e0')``. Used for producing canonical values for attributes - of an equivalence class. For example, ``Decimal('32.100')`` and - ``Decimal('0.321000e+2')`` both normalize to the equivalent value - ``Decimal('32.1')``. + Used for producing canonical values of an equivalence + class within either the current context or the specified context. + + This has the same semantics as the unary plus operation, except that if + the final result is finite it is reduced to its simplest form, with all + trailing zeros removed and its sign preserved. That is, while the + coefficient is non-zero and a multiple of ten the coefficient is divided + by ten and the exponent is incremented by 1. Otherwise (the coefficient is + zero) the exponent is set to 0. In all cases the sign is unchanged. + + For example, ``Decimal('32.100')`` and ``Decimal('0.321000e+2')`` both + normalize to the equivalent value ``Decimal('32.1')``. + + Note that rounding is applied *before* reducing to simplest form. + + In the latest versions of the specification, this operation is also known + as ``reduce``. .. method:: number_class(context=None) @@ -1385,10 +1396,10 @@ In addition to the three supplied contexts, new contexts can be created with the With three arguments, compute ``(x**y) % modulo``. For the three argument form, the following restrictions on the arguments hold: - - all three arguments must be integral - - ``y`` must be nonnegative - - at least one of ``x`` or ``y`` must be nonzero - - ``modulo`` must be nonzero and have at most 'precision' digits + - all three arguments must be integral + - ``y`` must be nonnegative + - at least one of ``x`` or ``y`` must be nonzero + - ``modulo`` must be nonzero and have at most 'precision' digits The value resulting from ``Context.power(x, y, modulo)`` is equal to the value that would be obtained by computing ``(x**y) @@ -1497,7 +1508,7 @@ are also included in the pure Python version for compatibility. The value is ``True``. Deprecated, because Python now always has threads. -.. deprecated:: 3.9 + .. deprecated:: 3.9 .. data:: HAVE_CONTEXTVAR @@ -2078,6 +2089,26 @@ representative: >>> [v.normalize() for v in values] [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] +Q. When does rounding occur in a computation? + +A. It occurs *after* the computation. The philosophy of the decimal +specification is that numbers are considered exact and are created +independent of the current context. They can even have greater +precision than current context. Computations process with those +exact inputs and then rounding (or other context operations) is +applied to the *result* of the computation:: + + >>> getcontext().prec = 5 + >>> pi = Decimal('3.1415926535') # More than 5 digits + >>> pi # All digits are retained + Decimal('3.1415926535') + >>> pi + 0 # Rounded after an addition + Decimal('3.1416') + >>> pi - Decimal('0.00005') # Subtract unrounded numbers, then round + Decimal('3.1415') + >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded + Decimal('3.1416') + Q. Some decimal values always print with exponential notation. Is there a way to get a non-exponential representation? diff --git a/Doc/library/development.rst b/Doc/library/development.rst index 9edce758688e2d..b1979b921e7d5f 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -8,8 +8,7 @@ The modules described in this chapter help you write software. For example, the :mod:`pydoc` module takes a module and generates documentation based on the module's contents. The :mod:`doctest` and :mod:`unittest` modules contains frameworks for writing unit tests that automatically exercise code and verify -that the expected output is produced. :program:`2to3` can translate Python 2.x -source code into valid Python 3.x code. +that the expected output is produced. The list of modules described in this chapter is: @@ -23,5 +22,4 @@ The list of modules described in this chapter is: unittest.rst unittest.mock.rst unittest.mock-examples.rst - 2to3.rst test.rst diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst index 977735990ffe92..5b8a9bd1908456 100644 --- a/Doc/library/devmode.rst +++ b/Doc/library/devmode.rst @@ -16,7 +16,7 @@ setting the :envvar:`PYTHONDEVMODE` environment variable to ``1``. See also :ref:`Python debug build `. Effects of the Python Development Mode -====================================== +-------------------------------------- Enabling the Python Development Mode is similar to the following command, but with additional effects described below:: @@ -59,8 +59,9 @@ Effects of the Python Development Mode: ``default``. * Call :func:`faulthandler.enable` at Python startup to install handlers for - the :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and - :const:`SIGILL` signals to dump the Python traceback on a crash. + the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, + :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` and + :const:`~signal.SIGILL` signals to dump the Python traceback on a crash. It behaves as if the :option:`-X faulthandler <-X>` command line option is used or if the :envvar:`PYTHONFAULTHANDLER` environment variable is set to @@ -81,7 +82,7 @@ Effects of the Python Development Mode: ignored for empty strings. * The :class:`io.IOBase` destructor logs ``close()`` exceptions. -* Set the :attr:`~sys.flags.dev_mode` attribute of :attr:`sys.flags` to +* Set the :attr:`~sys.flags.dev_mode` attribute of :data:`sys.flags` to ``True``. The Python Development Mode does not enable the :mod:`tracemalloc` module by @@ -107,7 +108,7 @@ value can be read from :data:`sys.flags.dev_mode `. ResourceWarning Example -======================= +----------------------- Example of a script counting the number of lines of the text file specified in the command line:: @@ -171,7 +172,7 @@ application more deterministic and more reliable. Bad file descriptor error example -================================= +--------------------------------- Script displaying the first line of itself:: @@ -198,7 +199,7 @@ descriptor" error when finalizing the file object: .. code-block:: shell-session - $ python script.py + $ python -X dev script.py import os script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='script.py' mode='r' encoding='UTF-8'> main() diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index 53f98c1018988f..191e0da12103fa 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -27,15 +27,15 @@ functions for creating simple modal dialogs to get a value from the user. The base class for custom dialogs. - .. method:: body(master) + .. method:: body(master) - Override to construct the dialog's interface and return the widget that - should have initial focus. + Override to construct the dialog's interface and return the widget that + should have initial focus. - .. method:: buttonbox() + .. method:: buttonbox() - Default behaviour adds OK and Cancel buttons. Override for custom button - layouts. + Default behaviour adds OK and Cancel buttons. Override for custom button + layouts. diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 5ee1f4a02c6816..9abf19557f989c 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -171,9 +171,12 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. expressed in the ISO 8601 format. If not specified, the strings default to blanks. + >>> import sys + >>> from difflib import * >>> s1 = ['bacon\n', 'eggs\n', 'ham\n', 'guido\n'] >>> s2 = ['python\n', 'eggy\n', 'hamster\n', 'guido\n'] - >>> sys.stdout.writelines(context_diff(s1, s2, fromfile='before.py', tofile='after.py')) + >>> sys.stdout.writelines(context_diff(s1, s2, fromfile='before.py', + ... tofile='after.py')) *** before.py --- after.py *************** @@ -294,13 +297,12 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. For inputs that do not have trailing newlines, set the *lineterm* argument to ``""`` so that the output will be uniformly newline free. - The context diff format normally has a header for filenames and modification + The unified diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for *fromfile*, *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally expressed in the ISO 8601 format. If not specified, the strings default to blanks. - >>> s1 = ['bacon\n', 'eggs\n', 'ham\n', 'guido\n'] >>> s2 = ['python\n', 'eggy\n', 'hamster\n', 'guido\n'] >>> sys.stdout.writelines(unified_diff(s1, s2, fromfile='before.py', tofile='after.py')) @@ -570,8 +572,8 @@ The :class:`SequenceMatcher` class has this constructor: The three methods that return the ratio of matching to total characters can give different results due to differing levels of approximation, although -:meth:`quick_ratio` and :meth:`real_quick_ratio` are always at least as large as -:meth:`ratio`: +:meth:`~SequenceMatcher.quick_ratio` and :meth:`~SequenceMatcher.real_quick_ratio` +are always at least as large as :meth:`~SequenceMatcher.ratio`: >>> s = SequenceMatcher(None, "abcd", "bcde") >>> s.ratio() @@ -593,15 +595,15 @@ This example compares two strings, considering blanks to be "junk": ... "private Thread currentThread;", ... "private volatile Thread currentThread;") -:meth:`ratio` returns a float in [0, 1], measuring the similarity of the -sequences. As a rule of thumb, a :meth:`ratio` value over 0.6 means the +:meth:`~SequenceMatcher.ratio` returns a float in [0, 1], measuring the similarity of the +sequences. As a rule of thumb, a :meth:`~SequenceMatcher.ratio` value over 0.6 means the sequences are close matches: >>> print(round(s.ratio(), 3)) 0.866 If you're only interested in where the sequences match, -:meth:`get_matching_blocks` is handy: +:meth:`~SequenceMatcher.get_matching_blocks` is handy: >>> for block in s.get_matching_blocks(): ... print("a[%d] and b[%d] match for %d elements" % block) @@ -609,12 +611,12 @@ If you're only interested in where the sequences match, a[8] and b[17] match for 21 elements a[29] and b[38] match for 0 elements -Note that the last tuple returned by :meth:`get_matching_blocks` is always a -dummy, ``(len(a), len(b), 0)``, and this is the only case in which the last +Note that the last tuple returned by :meth:`~SequenceMatcher.get_matching_blocks` +is always a dummy, ``(len(a), len(b), 0)``, and this is the only case in which the last tuple element (number of elements matched) is ``0``. If you want to know how to change the first sequence into the second, use -:meth:`get_opcodes`: +:meth:`~SequenceMatcher.get_opcodes`: >>> for opcode in s.get_opcodes(): ... print("%6s a[%d:%d] b[%d:%d]" % opcode) @@ -689,7 +691,7 @@ Differ Example This example compares two texts. First we set up the texts, sequences of individual single-line strings ending with newlines (such sequences can also be -obtained from the :meth:`~io.BaseIO.readlines` method of file-like objects): +obtained from the :meth:`~io.IOBase.readlines` method of file-like objects): >>> text1 = ''' 1. Beautiful is better than ugly. ... 2. Explicit is better than implicit. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 248743b8fa0a87..0d93bc9f5da774 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -42,27 +42,70 @@ interpreter. bytecode to specialize it for different runtime conditions. The adaptive bytecode can be shown by passing ``adaptive=True``. + .. versionchanged:: 3.12 + The argument of a jump is the offset of the target instruction relative + to the instruction that appears immediately after the jump instruction's + :opcode:`CACHE` entries. + + As a consequence, the presence of the :opcode:`CACHE` instructions is + transparent for forward jumps but needs to be taken into account when + reasoning about backward jumps. + + .. versionchanged:: 3.13 + The output shows logical labels rather than instruction offsets + for jump targets and exception handlers. The ``-O`` command line + option and the ``show_offsets`` argument were added. -Example: Given the function :func:`myfunc`:: +Example: Given the function :func:`!myfunc`:: def myfunc(alist): return len(alist) the following command can be used to display the disassembly of -:func:`myfunc`: +:func:`!myfunc`: .. doctest:: >>> dis.dis(myfunc) - 2 0 RESUME 0 + 2 RESUME 0 - 3 2 LOAD_GLOBAL 1 (NULL + len) - 12 LOAD_FAST 0 (alist) - 14 CALL 1 - 22 RETURN_VALUE + 3 LOAD_GLOBAL 1 (len + NULL) + LOAD_FAST 0 (alist) + CALL 1 + RETURN_VALUE (The "2" is a line number). +.. _dis-cli: + +Command-line interface +---------------------- + +The :mod:`dis` module can be invoked as a script from the command line: + +.. code-block:: sh + + python -m dis [-h] [-C] [-O] [infile] + +The following options are accepted: + +.. program:: dis + +.. cmdoption:: -h, --help + + Display usage and exit. + +.. cmdoption:: -C, --show-caches + + Show inline caches. + +.. cmdoption:: -O, --show-offsets + + Show offsets of instructions. + +If :file:`infile` is specified, its disassembled code will be written to stdout. +Otherwise, disassembly is performed on compiled source code recieved from stdin. + Bytecode analysis ----------------- @@ -73,7 +116,7 @@ The bytecode analysis API allows pieces of Python code to be wrapped in a code. .. class:: Bytecode(x, *, first_line=None, current_offset=None,\ - show_caches=False, adaptive=False) + show_caches=False, adaptive=False, show_offsets=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -98,6 +141,9 @@ code. If *adaptive* is ``True``, :meth:`.dis` will display specialized bytecode that may be different from the original bytecode. + If *show_offsets* is ``True``, :meth:`.dis` will include instruction + offsets in the output. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -188,9 +234,9 @@ operation is being performed, so the intermediate analysis object isn't useful: For a module, it disassembles all functions. For a class, it disassembles all methods (including class and static methods). For a code object or sequence of raw bytecode, it prints one line per bytecode instruction. - It also recursively disassembles nested code objects (the code of - comprehensions, generator expressions and nested functions, and the code - used for building nested classes). + It also recursively disassembles nested code objects. These can include + generator expressions, nested functions, the bodies of nested classes, + and the code objects used for :ref:`annotation scopes `. Strings are first compiled to code objects with the :func:`compile` built-in function before being disassembled. If no object is provided, this function disassembles the last traceback. @@ -220,7 +266,8 @@ operation is being performed, so the intermediate analysis object isn't useful: Added the *show_caches* and *adaptive* parameters. -.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False) +.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False, + show_offset=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -235,9 +282,12 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter. .. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) - disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) + disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False, + show_offsets=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: @@ -262,6 +312,8 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter. .. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) @@ -294,8 +346,12 @@ operation is being performed, so the intermediate analysis object isn't useful: Line numbers can be decreasing. Before, they were always increasing. .. versionchanged:: 3.10 - The :pep:`626` ``co_lines`` method is used instead of the ``co_firstlineno`` - and ``co_lnotab`` attributes of the code object. + The :pep:`626` ``co_lines`` method is used instead of the + :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` + attributes of the code object. + + .. versionchanged:: 3.13 + Line numbers can be ``None`` for bytecode that does not map to source lines. .. function:: findlabels(code) @@ -318,6 +374,12 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.8 Added *jump* parameter. + .. versionchanged:: 3.13 + If ``oparg`` is omitted (or ``None``), the stack effect is now returned + for ``oparg=0``. Previously this was an error for opcodes that use their + arg. It is also no longer an error to pass an integer ``oparg`` when + the ``opcode`` does not use it; the ``oparg`` in this case is ignored. + .. _bytecodes: @@ -342,10 +404,25 @@ details of bytecode instructions as :class:`Instruction` instances: human readable name for operation + .. data:: baseopcode + + numeric code for the base operation if operation is specialized; + otherwise equal to :data:`opcode` + + + .. data:: baseopname + + human readable name for the base operation if operation is specialized; + otherwise equal to :data:`opname` + + .. data:: arg numeric argument to operation (if any), otherwise ``None`` + .. data:: oparg + + alias for :data:`arg` .. data:: argval @@ -363,9 +440,30 @@ details of bytecode instructions as :class:`Instruction` instances: start index of operation within bytecode sequence + .. data:: start_offset + + start index of operation within bytecode sequence, including prefixed + ``EXTENDED_ARG`` operations if present; otherwise equal to :data:`offset` + + + .. data:: cache_offset + + start index of the cache entries following the operation + + + .. data:: end_offset + + end index of the cache entries following the operation + + .. data:: starts_line - line started by this opcode (if any), otherwise ``None`` + ``True`` if this opcode starts a source line, otherwise ``False`` + + + .. data:: line_number + + source line number associated with this opcode (if any), otherwise ``None`` .. data:: is_jump_target @@ -373,6 +471,12 @@ details of bytecode instructions as :class:`Instruction` instances: ``True`` if other code jumps to here, otherwise ``False`` + .. data:: jump_target + + bytecode index of the jump target if this is a jump operation, + otherwise ``None`` + + .. data:: positions :class:`dis.Positions` object holding the @@ -384,6 +488,14 @@ details of bytecode instructions as :class:`Instruction` instances: Field ``positions`` is added. + .. versionchanged:: 3.13 + + Changed field ``starts_line``. + + Added fields ``start_offset``, ``cache_offset``, ``end_offset``, + ``baseopname``, ``baseopcode``, ``jump_target``, ``oparg``, and + ``line_number``. + .. class:: Positions @@ -428,6 +540,14 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. versionadded:: 3.12 +.. opcode:: END_SEND + + Implements ``del STACK[-2]``. + Used to clean up when a generator exits. + + .. versionadded:: 3.12 + + .. opcode:: COPY (i) Push the i-th item to the top of the stack without removing it from its original @@ -481,6 +601,9 @@ result back on the stack. Implements ``STACK[-1] = not STACK[-1]``. + .. versionchanged:: 3.13 + This instruction now requires an exact :class:`bool` operand. + .. opcode:: UNARY_INVERT @@ -500,6 +623,13 @@ result back on the stack. .. versionadded:: 3.5 +.. opcode:: TO_BOOL + + Implements ``STACK[-1] = bool(STACK[-1])``. + + .. versionadded:: 3.13 + + **Binary and in-place operations** Binary operations remove the top two items from the stack (``STACK[-1]`` and @@ -528,7 +658,7 @@ not have to be) the original ``STACK[-2]``. key = STACK.pop() container = STACK.pop() - STACK.append(container[index]) + STACK.append(container[key]) .. opcode:: STORE_SUBSCR @@ -622,8 +752,8 @@ not have to be) the original ``STACK[-2]``. .. versionadded:: 3.8 - .. versionchanged:: 3.11 - Exception representation on the stack now consist of one, not three, items. + .. versionchanged:: 3.11 + Exception representation on the stack now consist of one, not three, items. .. opcode:: CLEANUP_THROW @@ -712,6 +842,9 @@ iterations of the loop. .. versionchanged:: 3.12 oparg set to be the exception block depth, for efficient closing of generators. + .. versionchanged:: 3.13 + oparg is ``1`` if this instruction is part of a yield-from or await, and ``0`` + otherwise. .. opcode:: SETUP_ANNOTATIONS @@ -733,8 +866,8 @@ iterations of the loop. .. opcode:: RERAISE Re-raises the exception currently on top of the stack. If oparg is non-zero, - pops an additional value from the stack which is used to set ``f_lasti`` - of the current frame. + pops an additional value from the stack which is used to set + :attr:`~frame.f_lasti` of the current frame. .. versionadded:: 3.9 @@ -793,7 +926,7 @@ iterations of the loop. .. opcode:: LOAD_BUILD_CLASS - Pushes :func:`builtins.__build_class__` onto the stack. It is later called + Pushes :func:`!builtins.__build_class__` onto the stack. It is later called to construct a class. @@ -818,7 +951,7 @@ iterations of the loop. .. opcode:: MATCH_MAPPING If ``STACK[-1]`` is an instance of :class:`collections.abc.Mapping` (or, more - technically: if it has the :const:`Py_TPFLAGS_MAPPING` flag set in its + technically: if it has the :c:macro:`Py_TPFLAGS_MAPPING` flag set in its :c:member:`~PyTypeObject.tp_flags`), push ``True`` onto the stack. Otherwise, push ``False``. @@ -829,7 +962,7 @@ iterations of the loop. If ``STACK[-1]`` is an instance of :class:`collections.abc.Sequence` and is *not* an instance of :class:`str`/:class:`bytes`/:class:`bytearray` (or, more technically: if it has - the :const:`Py_TPFLAGS_SEQUENCE` flag set in its :c:member:`~PyTypeObject.tp_flags`), + the :c:macro:`Py_TPFLAGS_SEQUENCE` flag set in its :c:member:`~PyTypeObject.tp_flags`), push ``True`` onto the stack. Otherwise, push ``False``. .. versionadded:: 3.10 @@ -851,22 +984,23 @@ iterations of the loop. .. opcode:: STORE_NAME (namei) Implements ``name = STACK.pop()``. *namei* is the index of *name* in the attribute - :attr:`co_names` of the code object. The compiler tries to use - :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. + :attr:`~codeobject.co_names` of the :ref:`code object `. + The compiler tries to use :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. .. opcode:: DELETE_NAME (namei) - Implements ``del name``, where *namei* is the index into :attr:`co_names` - attribute of the code object. + Implements ``del name``, where *namei* is the index into :attr:`~codeobject.co_names` + attribute of the :ref:`code object `. .. opcode:: UNPACK_SEQUENCE (count) Unpacks ``STACK[-1]`` into *count* individual values, which are put onto the stack - right-to-left:: + right-to-left. Require there to be exactly *count* values.:: - STACK.extend(STACK.pop()[:count:-1]) + assert(len(STACK[-1]) == count) + STACK.extend(STACK.pop()[:-count-1:-1]) .. opcode:: UNPACK_EX (counts) @@ -896,7 +1030,8 @@ iterations of the loop. value = STACK.pop() obj.name = value - where *namei* is the index of name in :attr:`co_names`. + where *namei* is the index of name in :attr:`~codeobject.co_names` of the + :ref:`code object `. .. opcode:: DELETE_ATTR (namei) @@ -905,7 +1040,8 @@ iterations of the loop. obj = STACK.pop() del obj.name - where *namei* is the index of name into :attr:`co_names`. + where *namei* is the index of name into :attr:`~codeobject.co_names` of the + :ref:`code object `. .. opcode:: STORE_GLOBAL (namei) @@ -926,6 +1062,27 @@ iterations of the loop. .. opcode:: LOAD_NAME (namei) Pushes the value associated with ``co_names[namei]`` onto the stack. + The name is looked up within the locals, then the globals, then the builtins. + + +.. opcode:: LOAD_LOCALS + + Pushes a reference to the locals dictionary onto the stack. This is used + to prepare namespace dictionaries for :opcode:`LOAD_FROM_DICT_OR_DEREF` + and :opcode:`LOAD_FROM_DICT_OR_GLOBALS`. + + .. versionadded:: 3.12 + + +.. opcode:: LOAD_FROM_DICT_OR_GLOBALS (i) + + Pops a mapping off the stack and looks up the value for ``co_names[namei]``. + If the name is not found there, looks it up in the globals and then the builtins, + similar to :opcode:`LOAD_GLOBAL`. + This is used for loading global variables in + :ref:`annotation scopes ` within class bodies. + + .. versionadded:: 3.12 .. opcode:: BUILD_TUPLE (count) @@ -1029,7 +1186,8 @@ iterations of the loop. This bytecode distinguishes two cases: if ``STACK[-1]`` has a method with the correct name, the bytecode pushes the unbound method and ``STACK[-1]``. ``STACK[-1]`` will be used as the first argument (``self``) by :opcode:`CALL` - when calling the unbound method. Otherwise, ``NULL`` and the object returned by + or :opcode:`CALL_KW` when calling the unbound method. + Otherwise, ``NULL`` and the object returned by the attribute lookup are pushed. .. versionchanged:: 3.12 @@ -1039,15 +1197,21 @@ iterations of the loop. .. opcode:: LOAD_SUPER_ATTR (namei) - This opcode implements :func:`super` (e.g. ``super().method()`` and - ``super().attr``). It works the same as :opcode:`LOAD_ATTR`, except that - ``namei`` is shifted left by 2 bits instead of 1, and instead of expecting a - single receiver on the stack, it expects three objects (from top of stack - down): ``self`` (the first argument to the current method), ``cls`` (the - class within which the current method was defined), and the global ``super``. + This opcode implements :func:`super`, both in its zero-argument and + two-argument forms (e.g. ``super().method()``, ``super().attr`` and + ``super(cls, self).method()``, ``super(cls, self).attr``). + + It pops three values from the stack (from top of stack down): + - ``self``: the first argument to the current method + - ``cls``: the class within which the current method was defined + - the global ``super`` + + With respect to its argument, it works similarly to :opcode:`LOAD_ATTR`, + except that ``namei`` is shifted left by 2 bits instead of 1. The low bit of ``namei`` signals to attempt a method load, as with - :opcode:`LOAD_ATTR`. + :opcode:`LOAD_ATTR`, which results in pushing ``None`` and the loaded method. + When it is unset a single value is pushed to the stack. The second-low bit of ``namei``, if set, means that this was a two-argument call to :func:`super` (unset means zero-argument). @@ -1058,7 +1222,12 @@ iterations of the loop. .. opcode:: COMPARE_OP (opname) Performs a Boolean operation. The operation name can be found in - ``cmp_op[opname]``. + ``cmp_op[opname >> 5]``. If the fifth-lowest bit of ``opname`` is set + (``opname & 16``), the result should be coerced to ``bool``. + + .. versionchanged:: 3.13 + The fifth-lowest bit of the oparg now indicates a forced conversion to + :class:`bool`. .. opcode:: IS_OP (invert) @@ -1122,6 +1291,9 @@ iterations of the loop. .. versionchanged:: 3.12 This is no longer a pseudo-instruction. + .. versionchanged:: 3.13 + This instruction now requires an exact :class:`bool` operand. + .. opcode:: POP_JUMP_IF_FALSE (delta) If ``STACK[-1]`` is false, increments the bytecode counter by *delta*. @@ -1135,6 +1307,9 @@ iterations of the loop. .. versionchanged:: 3.12 This is no longer a pseudo-instruction. + .. versionchanged:: 3.13 + This instruction now requires an exact :class:`bool` operand. + .. opcode:: POP_JUMP_IF_NOT_NONE (delta) If ``STACK[-1]`` is not ``None``, increments the bytecode counter by *delta*. @@ -1222,37 +1397,26 @@ iterations of the loop. .. versionadded:: 3.11 -.. opcode:: LOAD_CLOSURE (i) - - Pushes a reference to the cell contained in slot ``i`` of the "fast locals" - storage. The name of the variable is ``co_fastlocalnames[i]``. - - Note that ``LOAD_CLOSURE`` is effectively an alias for ``LOAD_FAST``. - It exists to keep bytecode a little more readable. - - .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. - - .. opcode:: LOAD_DEREF (i) Loads the cell contained in slot ``i`` of the "fast locals" storage. Pushes a reference to the object the cell contains on the stack. .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. -.. opcode:: LOAD_CLASSDEREF (i) +.. opcode:: LOAD_FROM_DICT_OR_DEREF (i) - Much like :opcode:`LOAD_DEREF` but first checks the locals dictionary before - consulting the cell. This is used for loading free variables in class - bodies. + Pops a mapping off the stack and looks up the name associated with + slot ``i`` of the "fast locals" storage in this mapping. + If the name is not found there, loads it from the cell contained in + slot ``i``, similar to :opcode:`LOAD_DEREF`. This is used for loading + free variables in class bodies (which previously used + :opcode:`!LOAD_CLASSDEREF`) and in + :ref:`annotation scopes ` within class bodies. - .. versionadded:: 3.4 - - .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + .. versionadded:: 3.12 .. opcode:: STORE_DEREF (i) @@ -1261,7 +1425,7 @@ iterations of the loop. storage. .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. .. opcode:: DELETE_DEREF (i) @@ -1272,7 +1436,7 @@ iterations of the loop. .. versionadded:: 3.2 .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. + ``i`` is no longer offset by the length of :attr:`~codeobject.co_varnames`. .. opcode:: COPY_FREE_VARS (n) @@ -1297,25 +1461,14 @@ iterations of the loop. .. opcode:: CALL (argc) - Calls a callable object with the number of arguments specified by ``argc``, - including the named arguments specified by the preceding - :opcode:`KW_NAMES`, if any. - On the stack are (in ascending order), either: - - * NULL - * The callable - * The positional arguments - * The named arguments - - or: + Calls a callable object with the number of arguments specified by ``argc``. + On the stack are (in ascending order): * The callable - * ``self`` + * ``self`` or ``NULL`` * The remaining positional arguments - * The named arguments - ``argc`` is the total of the positional and named arguments, excluding - ``self`` when a ``NULL`` is not present. + ``argc`` is the total of the positional arguments, excluding ``self``. ``CALL`` pops all arguments and the callable object off the stack, calls the callable object with those arguments, and pushes the return value @@ -1323,6 +1476,33 @@ iterations of the loop. .. versionadded:: 3.11 + .. versionchanged:: 3.13 + The callable now always appears at the same position on the stack. + + .. versionchanged:: 3.13 + Calls with keyword arguments are now handled by :opcode:`CALL_KW`. + + +.. opcode:: CALL_KW (argc) + + Calls a callable object with the number of arguments specified by ``argc``, + including one or more named arguments. On the stack are (in ascending order): + + * The callable + * ``self`` or ``NULL`` + * The remaining positional arguments + * The named arguments + * A :class:`tuple` of keyword argument names + + ``argc`` is the total of the positional and named arguments, excluding ``self``. + The length of the tuple of keyword argument names is the number of named arguments. + + ``CALL_KW`` pops all arguments, the keyword names, and the callable object + off the stack, calls the callable object with those arguments, and pushes the + return value returned by the callable object. + + .. versionadded:: 3.13 + .. opcode:: CALL_FUNCTION_EX (flags) @@ -1348,30 +1528,35 @@ iterations of the loop. .. versionadded:: 3.11 -.. opcode:: KW_NAMES (consti) +.. opcode:: MAKE_FUNCTION - Prefixes :opcode:`CALL`. - Stores a reference to ``co_consts[consti]`` into an internal variable - for use by :opcode:`CALL`. ``co_consts[consti]`` must be a tuple of strings. + Pushes a new function object on the stack built from the code object at ``STACK[1]``. - .. versionadded:: 3.11 + .. versionchanged:: 3.10 + Flag value ``0x04`` is a tuple of strings instead of dictionary + + .. versionchanged:: 3.11 + Qualified name at ``STACK[-1]`` was removed. + .. versionchanged:: 3.13 + Extra function attributes on the stack, signaled by oparg flags, were + removed. They now use :opcode:`SET_FUNCTION_ATTRIBUTE`. -.. opcode:: MAKE_FUNCTION (flags) - Pushes a new function object on the stack. From bottom to top, the consumed - stack must consist of values if the argument carries a specified flag value +.. opcode:: SET_FUNCTION_ATTRIBUTE (flag) + + Sets an attribute on a function object. Expects the function at ``STACK[-1]`` + and the attribute value to set at ``STACK[-2]``; consumes both and leaves the + function at ``STACK[-1]``. The flag determines which attribute to set: * ``0x01`` a tuple of default values for positional-only and positional-or-keyword parameters in positional order * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure - * the code associated with the function (at ``STACK[-2]``) - * the :term:`qualified name` of the function (at ``STACK[-1]``) - .. versionchanged:: 3.10 - Flag value ``0x04`` is a tuple of strings instead of dictionary + .. versionadded:: 3.13 + .. opcode:: BUILD_SLICE (argc) @@ -1401,26 +1586,47 @@ iterations of the loop. an argument from two-byte to four-byte. -.. opcode:: FORMAT_VALUE (flags) +.. opcode:: CONVERT_VALUE (oparg) - Used for implementing formatted literal strings (f-strings). Pops - an optional *fmt_spec* from the stack, then a required *value*. - *flags* is interpreted as follows: + Convert value to a string, depending on ``oparg``:: - * ``(flags & 0x03) == 0x00``: *value* is formatted as-is. - * ``(flags & 0x03) == 0x01``: call :func:`str` on *value* before - formatting it. - * ``(flags & 0x03) == 0x02``: call :func:`repr` on *value* before - formatting it. - * ``(flags & 0x03) == 0x03``: call :func:`ascii` on *value* before - formatting it. - * ``(flags & 0x04) == 0x04``: pop *fmt_spec* from the stack and use - it, else use an empty *fmt_spec*. + value = STACK.pop() + result = func(value) + STACK.push(result) - Formatting is performed using :c:func:`PyObject_Format`. The - result is pushed on the stack. + * ``oparg == 1``: call :func:`str` on *value* + * ``oparg == 2``: call :func:`repr` on *value* + * ``oparg == 3``: call :func:`ascii` on *value* - .. versionadded:: 3.6 + Used for implementing formatted literal strings (f-strings). + + .. versionadded:: 3.13 + + +.. opcode:: FORMAT_SIMPLE + + Formats the value on top of stack:: + + value = STACK.pop() + result = value.__format__("") + STACK.push(result) + + Used for implementing formatted literal strings (f-strings). + + .. versionadded:: 3.13 + +.. opcode:: FORMAT_SPEC + + Formats the given value with the given format spec:: + + spec = STACK.pop() + value = STACK.pop() + result = value.__format__(spec) + STACK.push(result) + + Used for implementing formatted literal strings (f-strings). + + .. versionadded:: 3.13 .. opcode:: MATCH_CLASS (count) @@ -1441,11 +1647,12 @@ iterations of the loop. success (``True``) or failure (``False``). -.. opcode:: RESUME (where) +.. opcode:: RESUME (context) A no-op. Performs internal tracing, debugging and optimization checks. - The ``where`` operand marks where the ``RESUME`` occurs: + The ``context`` oparand consists of two parts. The lowest two bits + indicate where the ``RESUME`` occurs: * ``0`` The start of a function, which is neither a generator, coroutine nor an async generator @@ -1453,8 +1660,14 @@ iterations of the loop. * ``2`` After a ``yield from`` expression * ``3`` After an ``await`` expression + The next bit is ``1`` if the RESUME is at except-depth ``1``, and ``0`` + otherwise. + .. versionadded:: 3.11 + .. versionchanged:: 3.13 + The oparg value changed to include information about except-depth + .. opcode:: RETURN_GENERATOR @@ -1470,9 +1683,9 @@ iterations of the loop. Equivalent to ``STACK[-1] = STACK[-2].send(STACK[-1])``. Used in ``yield from`` and ``await`` statements. - If the call raises :exc:`StopIteration`, pop both items, push the - exception's ``value`` attribute, and increment the bytecode counter by - *delta*. + If the call raises :exc:`StopIteration`, pop the top value from the stack, + push the exception's ``value`` attribute, and increment the bytecode counter + by *delta*. .. versionadded:: 3.11 @@ -1483,8 +1696,8 @@ iterations of the loop. opcodes in the range [0,255] which don't use their argument and those that do (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). - If your application uses pseudo instructions, use the :data:`hasarg` - collection instead. + If your application uses pseudo instructions or specialized instructions, + use the :data:`hasarg` collection instead. .. versionchanged:: 3.6 Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` @@ -1495,35 +1708,90 @@ iterations of the loop. it is not true that comparison with ``HAVE_ARGUMENT`` indicates whether they use their arg. + .. deprecated:: 3.13 + Use :data:`hasarg` instead. .. opcode:: CALL_INTRINSIC_1 Calls an intrinsic function with one argument. Passes ``STACK[-1]`` as the argument and sets ``STACK[-1]`` to the result. Used to implement - functionality that is necessary but not performance critical. + functionality that is not performance critical. The operand determines which intrinsic function is called: - * ``0`` Not valid - * ``1`` Prints the argument to standard out. Used in the REPL. - * ``2`` Performs ``import *`` for the named module. - * ``3`` Extracts the return value from a ``StopIteration`` exception. - * ``4`` Wraps an aync generator value - * ``5`` Performs the unary ``+`` operation - * ``6`` Converts a list to a tuple + +-----------------------------------+-----------------------------------+ + | Operand | Description | + +===================================+===================================+ + | ``INTRINSIC_1_INVALID`` | Not valid | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_PRINT`` | Prints the argument to standard | + | | out. Used in the REPL. | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_IMPORT_STAR`` | Performs ``import *`` for the | + | | named module. | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_STOPITERATION_ERROR`` | Extracts the return value from a | + | | ``StopIteration`` exception. | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an aync generator value | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_UNARY_POSITIVE`` | Performs the unary ``+`` | + | | operation | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_LIST_TO_TUPLE`` | Converts a list to a tuple | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVAR`` | Creates a :class:`typing.TypeVar` | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_PARAMSPEC`` | Creates a | + | | :class:`typing.ParamSpec` | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVARTUPLE`` | Creates a | + | | :class:`typing.TypeVarTuple` | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_SUBSCRIPT_GENERIC`` | Returns :class:`typing.Generic` | + | | subscripted with the argument | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEALIAS`` | Creates a | + | | :class:`typing.TypeAliasType`; | + | | used in the :keyword:`type` | + | | statement. The argument is a tuple| + | | of the type alias's name, | + | | type parameters, and value. | + +-----------------------------------+-----------------------------------+ .. versionadded:: 3.12 .. opcode:: CALL_INTRINSIC_2 - Calls an intrinsic function with two arguments. Passes ``STACK[-2]``, ``STACK[-1]`` as the - arguments and sets ``STACK[-1]`` to the result. Used to implement functionality that is - necessary but not performance critical. + Calls an intrinsic function with two arguments. Used to implement functionality + that is not performance critical:: + + arg2 = STACK.pop() + arg1 = STACK.pop() + result = intrinsic2(arg1, arg2) + STACK.push(result) The operand determines which intrinsic function is called: - * ``0`` Not valid - * ``1`` Calculates the :exc:`ExceptionGroup` to raise from a ``try-except*``. + +----------------------------------------+-----------------------------------+ + | Operand | Description | + +========================================+===================================+ + | ``INTRINSIC_2_INVALID`` | Not valid | + +----------------------------------------+-----------------------------------+ + | ``INTRINSIC_PREP_RERAISE_STAR`` | Calculates the | + | | :exc:`ExceptionGroup` to raise | + | | from a ``try-except*``. | + +----------------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVAR_WITH_BOUND`` | Creates a :class:`typing.TypeVar` | + | | with a bound. | + +----------------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVAR_WITH_CONSTRAINTS`` | Creates a | + | | :class:`typing.TypeVar` with | + | | constraints. | + +----------------------------------------+-----------------------------------+ + | ``INTRINSIC_SET_FUNCTION_TYPE_PARAMS`` | Sets the ``__type_params__`` | + | | attribute of a function. | + +----------------------------------------+-----------------------------------+ .. versionadded:: 3.12 @@ -1571,6 +1839,17 @@ but are replaced by real opcodes or removed before bytecode is generated. Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. +.. opcode:: LOAD_CLOSURE (i) + + Pushes a reference to the cell contained in slot ``i`` of the "fast locals" + storage. + + Note that ``LOAD_CLOSURE`` is replaced with ``LOAD_FAST`` in the assembler. + + .. versionchanged:: 3.13 + This opcode is now a pseudo-instruction. + + .. opcode:: LOAD_METHOD Optimized unbound method lookup. Emitted as a ``LOAD_ATTR`` opcode @@ -1586,8 +1865,9 @@ These collections are provided for automatic introspection of bytecode instructions: .. versionchanged:: 3.12 - The collections now contain pseudo instructions as well. These are - opcodes with values ``>= MIN_PSEUDO_OPCODE``. + The collections now contain pseudo instructions and instrumented + instructions as well. These are opcodes with values ``>= MIN_PSEUDO_OPCODE`` + and ``>= MIN_INSTRUMENTED_OPCODE``. .. data:: opname @@ -1629,15 +1909,12 @@ instructions: Sequence of bytecodes that access an attribute by name. -.. data:: hasjrel +.. data:: hasjump - Sequence of bytecodes that have a relative jump target. - - -.. data:: hasjabs - - Sequence of bytecodes that have an absolute jump target. + Sequence of bytecodes that have a jump target. All jumps + are relative. + .. versionadded:: 3.13 .. data:: haslocal @@ -1653,3 +1930,20 @@ instructions: Sequence of bytecodes that set an exception handler. .. versionadded:: 3.12 + + +.. data:: hasjrel + + Sequence of bytecodes that have a relative jump target. + + .. deprecated:: 3.13 + All jumps are now relative. Use :data:`hasjump`. + + +.. data:: hasjabs + + Sequence of bytecodes that have an absolute jump target. + + .. deprecated:: 3.13 + All jumps are now relative. This list is empty. + diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index d6e4dca0860671..fa1b850c531346 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -143,13 +143,13 @@ Simple Usage: Checking Examples in Docstrings --------------------------------------------- The simplest way to start using doctest (but not necessarily the way you'll -continue to do it) is to end each module :mod:`M` with:: +continue to do it) is to end each module :mod:`!M` with:: if __name__ == "__main__": import doctest doctest.testmod() -:mod:`doctest` then examines docstrings in module :mod:`M`. +:mod:`!doctest` then examines docstrings in module :mod:`!M`. Running the module as a script causes the examples in the docstrings to get executed and verified:: @@ -277,13 +277,34 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. -In addition, if ``M.__test__`` exists and "is true", it must be a dict, and each +In addition, there are cases when you want tests to be part of a module but not part +of the help text, which requires that the tests not be included in the docstring. +Doctest looks for a module-level variable called ``__test__`` and uses it to locate other +tests. If ``M.__test__`` exists and is truthy, it must be a dict, and each entry maps a (string) name to a function object, class object, or string. Function and class object docstrings found from ``M.__test__`` are searched, and strings are treated as if they were docstrings. In output, a key ``K`` in -``M.__test__`` appears with name :: +``M.__test__`` appears with name ``M.__test__.K``. - .__test__.K +For example, place this block of code at the top of :file:`example.py`: + +.. code-block:: python + + __test__ = { + 'numbers': """ + >>> factorial(6) + 720 + + >>> [factorial(n) for n in range(6)] + [1, 1, 2, 6, 24, 120] + """ + } + +The value of ``example.__test__["numbers"]`` will be treated as a +docstring and all the tests inside it will be run. It is +important to note that the value can be mapped to a function, +class object, or module; if so, :mod:`!doctest` +searches them recursively for docstrings, which are then scanned for tests. Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. @@ -382,10 +403,10 @@ What's the Execution Context? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, each time :mod:`doctest` finds a docstring to test, it uses a -*shallow copy* of :mod:`M`'s globals, so that running tests doesn't change the -module's real globals, and so that one test in :mod:`M` can't leave behind +*shallow copy* of :mod:`!M`'s globals, so that running tests doesn't change the +module's real globals, and so that one test in :mod:`!M` can't leave behind crumbs that accidentally allow another test to work. This means examples can -freely use any names defined at top-level in :mod:`M`, and names defined earlier +freely use any names defined at top-level in :mod:`!M`, and names defined earlier in the docstring being run. Examples cannot see names defined in other docstrings. @@ -409,10 +430,10 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): File "", line 1, in - ValueError: list.remove(x): x not in list + ValueError: 42 is not in list -That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): -x not in list`` detail as shown. +That doctest succeeds if :exc:`ValueError` is raised, with the ``42 is not in list`` +detail as shown. The expected output for an exception must start with a traceback header, which may be either of the following two lines, indented the same as the first line of @@ -937,7 +958,8 @@ and :ref:`doctest-simple-testfile`. Optional argument *exclude_empty* defaults to false. If true, objects for which no doctests are found are excluded from consideration. The default is a backward - compatibility hack, so that code still using :meth:`doctest.master.summarize` in + compatibility hack, so that code still using + :meth:`doctest.master.summarize ` in conjunction with :func:`testmod` continues to get output for objects with no tests. The *exclude_empty* argument to the newer :class:`DocTestFinder` constructor defaults to true. @@ -976,7 +998,7 @@ As your collection of doctest'ed modules grows, you'll want a way to run all their doctests systematically. :mod:`doctest` provides two functions that can be used to create :mod:`unittest` test suites from modules and text files containing doctests. To integrate with :mod:`unittest` test discovery, include -a :func:`load_tests` function in your test module:: +a :ref:`load_tests ` function in your test module:: import unittest import doctest @@ -1090,19 +1112,24 @@ from text files and modules with doctests: :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* contains no docstrings instead of raising :exc:`ValueError`. +.. exception:: failureException + + When doctests which have been converted to unit tests by :func:`DocFileSuite` + or :func:`DocTestSuite` fail, this exception is raised showing the name of + the file containing the test and a (sometimes approximate) line number. Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out -of :class:`doctest.DocTestCase` instances, and :class:`DocTestCase` is a -subclass of :class:`unittest.TestCase`. :class:`DocTestCase` isn't documented +of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a +subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented here (it's an internal detail), but studying its code can answer questions about the exact details of :mod:`unittest` integration. Similarly, :func:`DocFileSuite` creates a :class:`unittest.TestSuite` out of -:class:`doctest.DocFileCase` instances, and :class:`DocFileCase` is a subclass -of :class:`DocTestCase`. +:class:`!doctest.DocFileCase` instances, and :class:`!DocFileCase` is a subclass +of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of -:class:`DocTestCase`. This is important for a subtle reason: when you run +:class:`!DocTestCase`. This is important for a subtle reason: when you run :mod:`doctest` functions yourself, you can control the :mod:`doctest` options in use directly, by passing option flags to :mod:`doctest` functions. However, if you're writing a :mod:`unittest` framework, :mod:`unittest` ultimately controls @@ -1123,14 +1150,14 @@ reporting flags specific to :mod:`unittest` support, via this function: section :ref:`doctest-options`. Only "reporting flags" can be used. This is a module-global setting, and affects all future doctests run by module - :mod:`unittest`: the :meth:`runTest` method of :class:`DocTestCase` looks at - the option flags specified for the test case when the :class:`DocTestCase` + :mod:`unittest`: the :meth:`!runTest` method of :class:`!DocTestCase` looks at + the option flags specified for the test case when the :class:`!DocTestCase` instance was constructed. If no reporting flags were specified (which is the - typical and expected case), :mod:`doctest`'s :mod:`unittest` reporting flags are + typical and expected case), :mod:`!doctest`'s :mod:`unittest` reporting flags are :ref:`bitwise ORed ` into the option flags, and the option flags so augmented are passed to the :class:`DocTestRunner` instance created to run the doctest. If any reporting flags were specified when the - :class:`DocTestCase` instance was constructed, :mod:`doctest`'s + :class:`!DocTestCase` instance was constructed, :mod:`!doctest`'s :mod:`unittest` reporting flags are ignored. The value of the :mod:`unittest` reporting flags in effect before the function @@ -1300,7 +1327,8 @@ Example Objects A dictionary mapping from option flags to ``True`` or ``False``, which is used to override default options for this example. Any option flags not contained in this dictionary are left at their default value (as specified by the - :class:`DocTestRunner`'s :attr:`optionflags`). By default, no options are set. + :class:`DocTestRunner`'s :ref:`optionflags `). + By default, no options are set. .. _doctest-doctestfinder: @@ -1409,6 +1437,27 @@ DocTestParser objects identifying this string, and is only used for error messages. +TestResults objects +^^^^^^^^^^^^^^^^^^^ + + +.. class:: TestResults(failed, attempted) + + .. attribute:: failed + + Number of failed tests. + + .. attribute:: attempted + + Number of attempted tests. + + .. attribute:: skipped + + Number of skipped tests. + + .. versionadded:: 3.13 + + .. _doctest-doctestrunner: DocTestRunner objects @@ -1427,7 +1476,7 @@ DocTestRunner objects passing a subclass of :class:`OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. First, an output - function can be passed to :meth:`TestRunner.run`; this function will be called + function can be passed to :meth:`run`; this function will be called with strings that should be displayed. It defaults to ``sys.stdout.write``. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods @@ -1448,6 +1497,10 @@ DocTestRunner objects runner compares expected output to actual output, and how it displays failures. For more information, see section :ref:`doctest-options`. + The test runner accumulates statistics. The aggregated number of attempted, + failed and skipped examples is also available via the :attr:`tries`, + :attr:`failures` and :attr:`skips` attributes. The :meth:`run` and + :meth:`summarize` methods return a :class:`TestResults` instance. :class:`DocTestParser` defines the following methods: @@ -1500,7 +1553,8 @@ DocTestRunner objects .. method:: run(test, compileflags=None, out=None, clear_globs=True) Run the examples in *test* (a :class:`DocTest` object), and display the - results using the writer function *out*. + results using the writer function *out*. Return a :class:`TestResults` + instance. The examples are run in the namespace ``test.globs``. If *clear_globs* is true (the default), then this namespace will be cleared after the test runs, @@ -1513,18 +1567,35 @@ DocTestRunner objects The output of each example is checked using the :class:`DocTestRunner`'s output checker, and the results are formatted by the - :meth:`DocTestRunner.report_\*` methods. + :meth:`!DocTestRunner.report_\*` methods. .. method:: summarize(verbose=None) Print a summary of all the test cases that have been run by this DocTestRunner, - and return a :term:`named tuple` ``TestResults(failed, attempted)``. + and return a :class:`TestResults` instance. The optional *verbose* argument controls how detailed the summary is. If the verbosity is not specified, then the :class:`DocTestRunner`'s verbosity is used. + :class:`DocTestParser` has the following attributes: + + .. attribute:: tries + + Number of attempted examples. + + .. attribute:: failures + + Number of failed examples. + + .. attribute:: skips + + Number of skipped examples. + + .. versionadded:: 3.13 + + .. _doctest-outputchecker: OutputChecker objects @@ -1671,12 +1742,12 @@ code under the debugger: module) of the object with the doctests of interest. The result is a string, containing the object's docstring converted to a Python script, as described for :func:`script_from_examples` above. For example, if module :file:`a.py` - contains a top-level function :func:`f`, then :: + contains a top-level function :func:`!f`, then :: import a, doctest print(doctest.testsource(a, "a.f")) - prints a script version of function :func:`f`'s docstring, with doctests + prints a script version of function :func:`!f`'s docstring, with doctests converted to code, and the rest placed in comments. diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index adbe6c1c7d29b8..aa0134412f3a60 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -150,7 +150,7 @@ Import this class from the :mod:`email.charset` module. .. method:: __str__() Returns *input_charset* as a string coerced to lower - case. :meth:`__repr__` is an alias for :meth:`__str__`. + case. :meth:`!__repr__` is an alias for :meth:`!__str__`. .. method:: __eq__(other) diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 5bef155a4af310..c4c322a82e1f44 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -367,7 +367,7 @@ Here are the methods of the :class:`Message` class: .. method:: get(name, failobj=None) Return the value of the named header field. This is identical to - :meth:`__getitem__` except that optional *failobj* is returned if the + :meth:`~object.__getitem__` except that optional *failobj* is returned if the named header is missing (defaults to ``None``). Here are some additional useful methods: diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 918fc55677e723..5b49339650f0e9 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -32,9 +32,9 @@ To find the handler, look for the following keys in the registry, stopping with the first one found: - * the string representing the full MIME type (``maintype/subtype``) - * the string representing the ``maintype`` - * the empty string + * the string representing the full MIME type (``maintype/subtype``) + * the string representing the ``maintype`` + * the empty string If none of these keys produce a handler, raise a :exc:`KeyError` for the full MIME type. @@ -55,11 +55,11 @@ look for the following keys in the registry, stopping with the first one found: - * the type itself (``typ``) - * the type's fully qualified name (``typ.__module__ + '.' + - typ.__qualname__``). - * the type's qualname (``typ.__qualname__``) - * the type's name (``typ.__name__``). + * the type itself (``typ``) + * the type's fully qualified name (``typ.__module__ + '.' + + typ.__qualname__``). + * the type's qualname (``typ.__qualname__``) + * the type's name (``typ.__name__``). If none of the above match, repeat all of the checks above for each of the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key @@ -132,15 +132,15 @@ Currently the email package provides only one concrete content manager, Add a :mailheader:`Content-Type` header with a ``maintype/subtype`` value. - * For ``str``, set the MIME ``maintype`` to ``text``, and set the - subtype to *subtype* if it is specified, or ``plain`` if it is not. - * For ``bytes``, use the specified *maintype* and *subtype*, or - raise a :exc:`TypeError` if they are not specified. - * For :class:`~email.message.EmailMessage` objects, set the maintype - to ``message``, and set the subtype to *subtype* if it is - specified or ``rfc822`` if it is not. If *subtype* is - ``partial``, raise an error (``bytes`` objects must be used to - construct ``message/partial`` parts). + * For ``str``, set the MIME ``maintype`` to ``text``, and set the + subtype to *subtype* if it is specified, or ``plain`` if it is not. + * For ``bytes``, use the specified *maintype* and *subtype*, or + raise a :exc:`TypeError` if they are not specified. + * For :class:`~email.message.EmailMessage` objects, set the maintype + to ``message``, and set the subtype to *subtype* if it is + specified or ``rfc822`` if it is not. If *subtype* is + ``partial``, raise an error (``bytes`` objects must be used to + construct ``message/partial`` parts). If *charset* is provided (which is valid only for ``str``), encode the string to bytes using the specified character set. The default is @@ -155,14 +155,14 @@ Currently the email package provides only one concrete content manager, ``7bit`` for an input that contains non-ASCII values), raise a :exc:`ValueError`. - * For ``str`` objects, if *cte* is not set use heuristics to - determine the most compact encoding. - * For :class:`~email.message.EmailMessage`, per :rfc:`2046`, raise - an error if a *cte* of ``quoted-printable`` or ``base64`` is - requested for *subtype* ``rfc822``, and for any *cte* other than - ``7bit`` for *subtype* ``external-body``. For - ``message/rfc822``, use ``8bit`` if *cte* is not specified. For - all other values of *subtype*, use ``7bit``. + * For ``str`` objects, if *cte* is not set use heuristics to + determine the most compact encoding. + * For :class:`~email.message.EmailMessage`, per :rfc:`2046`, raise + an error if a *cte* of ``quoted-printable`` or ``base64`` is + requested for *subtype* ``rfc822``, and for any *cte* other than + ``7bit`` for *subtype* ``external-body``. For + ``message/rfc822``, use ``8bit`` if *cte* is not specified. For + all other values of *subtype*, use ``7bit``. .. note:: A *cte* of ``binary`` does not actually work correctly yet. The ``EmailMessage`` object as modified by ``set_content`` is diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 5d68b104f3a45c..3bd377e33f6c15 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -25,7 +25,7 @@ is especially true for :mimetype:`image/\*` and :mimetype:`text/\*` type message containing binary data. The :mod:`email` package provides some convenient encoders in its -:mod:`encoders` module. These encoders are actually used by the +:mod:`~email.encoders` module. These encoders are actually used by the :class:`~email.mime.audio.MIMEAudio` and :class:`~email.mime.image.MIMEImage` class constructors to provide default encodings. All encoder functions take exactly one argument, the message object to encode. They usually extract the diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 194a98696f437d..56aea6598b8615 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -58,6 +58,15 @@ The following exception classes are defined in the :mod:`email.errors` module: :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. :class:`~email.mime.image.MIMEImage`). +.. exception:: MessageDefect() + + This is the base class for all defects found when parsing email messages. + It is derived from :exc:`ValueError`. + +.. exception:: HeaderDefect() + + This is the base class for all defects found when parsing email headers. + It is derived from :exc:`MessageDefect`. Here is the list of the defects that the :class:`~email.parser.FeedParser` can find while parsing messages. Note that the defects are added to the message diff --git a/Doc/library/email.examples.rst b/Doc/library/email.examples.rst index fc964622809d0e..492a8354d8bf85 100644 --- a/Doc/library/email.examples.rst +++ b/Doc/library/email.examples.rst @@ -55,11 +55,11 @@ Up to the prompt, the output from the above is: To: Penelope Pussycat , Fabrette Pussycat From: Pepé Le Pew - Subject: Ayons asperges pour le déjeuner + Subject: Pourquoi pas des asperges pour ce midi ? Salut! - Cela ressemble à un excellent recipie[1] déjeuner. + Cette recette [1] sera sûrement un très bon repas. .. rubric:: Footnotes diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index 34ad7b7f200af3..afa0038ea2d6c4 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -10,8 +10,8 @@ One of the most common tasks is to generate the flat (serialized) version of the email message represented by a message object structure. You will need to -do this if you want to send your message via :meth:`smtplib.SMTP.sendmail` or -the :mod:`nntplib` module, or print the message on the console. Taking a +do this if you want to send your message via :meth:`smtplib.SMTP.sendmail`, +or print the message on the console. Taking a message object structure and producing a serialized representation is the job of the generator classes. @@ -274,9 +274,9 @@ in with information about the part. .. rubric:: Footnotes .. [#] This statement assumes that you use the appropriate setting for - ``unixfrom``, and that there are no :mod:`policy` settings calling for + ``unixfrom``, and that there are no :mod:`email.policy` settings calling for automatic adjustments (for example, - :attr:`~email.policy.Policy.refold_source` must be ``none``, which is + :attr:`~email.policy.EmailPolicy.refold_source` must be ``none``, which is *not* the default). It is also not 100% true, since if the message does not conform to the RFC standards occasionally information about the exact original text is lost during parsing error recovery. It is a goal diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 5e0509f4181199..f58d93da6ed687 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -67,7 +67,7 @@ message objects. with the base :class:`~email.message.Message` class *maxheaderlen* is accepted, but defaults to ``None``, which means that by default the line length is controlled by the - :attr:`~email.policy.EmailPolicy.max_line_length` of the policy. The + :attr:`~email.policy.Policy.max_line_length` of the policy. The *policy* argument may be used to override the default policy obtained from the message instance. This can be used to control some of the formatting produced by the method, since the specified *policy* will be @@ -213,7 +213,7 @@ message objects. del msg['subject'] msg['subject'] = 'Python roolz!' - If the :mod:`policy` defines certain headers to be unique (as the standard + If the :mod:`policy ` defines certain headers to be unique (as the standard policies do), this method may raise a :exc:`ValueError` when an attempt is made to assign a value to such a header when one already exists. This behavior is intentional for consistency's sake, but do not depend on it @@ -247,7 +247,7 @@ message objects. .. method:: get(name, failobj=None) Return the value of the named header field. This is identical to - :meth:`__getitem__` except that optional *failobj* is returned if the + :meth:`~object.__getitem__` except that optional *failobj* is returned if the named header is missing (*failobj* defaults to ``None``). @@ -378,7 +378,7 @@ message objects. deprecated. Note that existing parameter values of headers may be accessed through - the :attr:`~email.headerregistry.BaseHeader.params` attribute of the + the :attr:`~email.headerregistry.ParameterizedMIMEHeader.params` attribute of the header value (for example, ``msg['Content-Type'].params['charset']``). .. versionchanged:: 3.4 ``replace`` keyword was added. @@ -691,7 +691,7 @@ message objects. .. method:: clear_content() - Remove the payload and all of the :exc:`Content-` headers, leaving + Remove the payload and all of the :mailheader:`!Content-` headers, leaving all other headers intact and in their original order. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index d9a61616bbbdfb..dda0466a6afa7d 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -39,9 +39,9 @@ returns the root object when you close the parser. Note that the parser can be extended in limited ways, and of course you can implement your own parser completely from scratch. All of the logic that connects the :mod:`email` package's bundled parser and the -:class:`~email.message.EmailMessage` class is embodied in the :mod:`policy` +:class:`~email.message.EmailMessage` class is embodied in the :class:`~email.policy.Policy` class, so a custom parser can create message object trees any way it finds -necessary by implementing custom versions of the appropriate :mod:`policy` +necessary by implementing custom versions of the appropriate :class:`!Policy` methods. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 2439dee676c9b0..fd47dd0dc5df36 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -557,17 +557,17 @@ more closely to the RFCs relevant to their domains. With all of these :class:`EmailPolicies <.EmailPolicy>`, the effective API of the email package is changed from the Python 3.2 API in the following ways: - * Setting a header on a :class:`~email.message.Message` results in that - header being parsed and a header object created. +* Setting a header on a :class:`~email.message.Message` results in that + header being parsed and a header object created. - * Fetching a header value from a :class:`~email.message.Message` results - in that header being parsed and a header object created and - returned. +* Fetching a header value from a :class:`~email.message.Message` results + in that header being parsed and a header object created and + returned. - * Any header object, or any header that is refolded due to the - policy settings, is folded using an algorithm that fully implements the - RFC folding algorithms, including knowing where encoded words are required - and allowed. +* Any header object, or any header that is refolded due to the + policy settings, is folded using an algorithm that fully implements the + RFC folding algorithms, including knowing where encoded words are required + and allowed. From the application view, this means that any header obtained through the :class:`~email.message.EmailMessage` is a header object with extra diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 816fae991d24cb..3a6039004fcaae 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -15,7 +15,7 @@ The :mod:`email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as -:mod:`smtplib` and :mod:`nntplib`. The :mod:`email` package attempts to be as +:mod:`smtplib`. The :mod:`email` package attempts to be as RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. @@ -141,9 +141,6 @@ Legacy API: Module :mod:`imaplib` IMAP (Internet Message Access Protocol) client - Module :mod:`nntplib` - NNTP (Net News Transport Protocol) client - Module :mod:`mailbox` Tools for creating, reading, and managing collections of messages on disk using a variety standard formats. diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index d7f89cf96368b5..de3b93f5e61073 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -61,7 +61,7 @@ By default, ``pip`` is installed into the current virtual environment active virtual environment). The installation location can be controlled through two additional command line options: -* ``--root

``: Installs ``pip`` relative to the given root directory +* :samp:`--root {dir}`: Installs ``pip`` relative to the given root directory rather than the root of the currently active virtual environment (if any) or the default root for the current Python installation. * ``--user``: Installs ``pip`` into the user site packages directory rather diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 582e06261afd72..20222bfb3611ab 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -165,7 +165,7 @@ Data Types to subclass *EnumType* -- see :ref:`Subclassing EnumType ` for details. - *EnumType* is responsible for setting the correct :meth:`!__repr__`, + ``EnumType`` is responsible for setting the correct :meth:`!__repr__`, :meth:`!__str__`, :meth:`!__format__`, and :meth:`!__reduce__` methods on the final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. @@ -198,11 +198,12 @@ Data Types >>> some_var = Color.RED >>> some_var in Color True + >>> Color.RED.value in Color + True - .. note:: + .. versionchanged:: 3.12 - In Python 3.12 it will be possible to check for member values and not - just members; until then, a ``TypeError`` will be raised if a + Before Python 3.12, a ``TypeError`` is raised if a non-Enum-member is used in a containment check. .. method:: EnumType.__dir__(cls) @@ -234,6 +235,10 @@ Data Types >>> len(Color) 3 + .. attribute:: EnumType.__members__ + + Returns a mapping of every enum name to its member, including aliases + .. method:: EnumType.__reversed__(cls) Returns each member in *cls* in reverse definition order:: @@ -241,6 +246,20 @@ Data Types >>> list(reversed(Color)) [, , ] + .. method:: EnumType._add_alias_ + + Adds a new name as an alias to an existing member. Raises a + :exc:`NameError` if the name is already assigned to a different member. + + .. method:: EnumType._add_value_alias_ + + Adds a new value as an alias to an existing member. Raises a + :exc:`ValueError` if the value is already linked with a different member. + + .. versionadded:: 3.11 + + Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. + .. class:: Enum @@ -401,23 +420,23 @@ Data Types .. class:: IntEnum - *IntEnum* is the same as *Enum*, but its members are also integers and can be + *IntEnum* is the same as :class:`Enum`, but its members are also integers and can be used anywhere that an integer can be used. If any integer operation is performed with an *IntEnum* member, the resulting value loses its enumeration status. >>> from enum import IntEnum - >>> class Numbers(IntEnum): + >>> class Number(IntEnum): ... ONE = 1 ... TWO = 2 ... THREE = 3 ... - >>> Numbers.THREE - - >>> Numbers.ONE + Numbers.TWO + >>> Number.THREE + + >>> Number.ONE + Number.TWO 3 - >>> Numbers.THREE + 5 + >>> Number.THREE + 5 8 - >>> Numbers.THREE == 3 + >>> Number.THREE == 3 True .. note:: @@ -432,7 +451,7 @@ Data Types .. class:: StrEnum - *StrEnum* is the same as *Enum*, but its members are also strings and can be used + ``StrEnum`` is the same as :class:`Enum`, but its members are also strings and can be used in most of the same places that a string can be used. The result of any string operation performed on or with a *StrEnum* member is not part of the enumeration. @@ -544,7 +563,7 @@ Data Types .. method:: __invert__(self): - Returns all the flags in *type(self)* that are not in self:: + Returns all the flags in *type(self)* that are not in *self*:: >>> ~white @@ -571,7 +590,7 @@ Data Types .. class:: IntFlag - *IntFlag* is the same as *Flag*, but its members are also integers and can be + ``IntFlag`` is the same as :class:`Flag`, but its members are also integers and can be used anywhere that an integer can be used. >>> from enum import IntFlag, auto @@ -591,12 +610,12 @@ Data Types >>> Color.RED + 2 3 - If a *Flag* operation is performed with an *IntFlag* member and: + If a :class:`Flag` operation is performed with an *IntFlag* member and: - * the result is a valid *IntFlag*: an *IntFlag* is returned - * the result is not a valid *IntFlag*: the result depends on the *FlagBoundary* setting + * the result is a valid *IntFlag*: an *IntFlag* is returned + * the result is not a valid *IntFlag*: the result depends on the :class:`FlagBoundary` setting - The *repr()* of unnamed zero-valued flags has changed. It is now: + The :func:`repr()` of unnamed zero-valued flags has changed. It is now: >>> Color(0) @@ -621,8 +640,8 @@ Data Types :class:`!ReprEnum` uses the :meth:`repr() ` of :class:`Enum`, but the :class:`str() ` of the mixed-in data type: - * :meth:`!int.__str__` for :class:`IntEnum` and :class:`IntFlag` - * :meth:`!str.__str__` for :class:`StrEnum` + * :meth:`!int.__str__` for :class:`IntEnum` and :class:`IntFlag` + * :meth:`!str.__str__` for :class:`StrEnum` Inherit from :class:`!ReprEnum` to keep the :class:`str() ` / :func:`format` of the mixed-in data type instead of using the @@ -692,7 +711,7 @@ Data Types .. class:: FlagBoundary - *FlagBoundary* controls how out-of-range values are handled in *Flag* and its + ``FlagBoundary`` controls how out-of-range values are handled in :class:`Flag` and its subclasses. .. attribute:: STRICT @@ -715,7 +734,7 @@ Data Types .. attribute:: CONFORM - Out-of-range values have invalid values removed, leaving a valid *Flag* + Out-of-range values have invalid values removed, leaving a valid :class:`Flag` value:: >>> from enum import Flag, CONFORM, auto @@ -729,7 +748,7 @@ Data Types .. attribute:: EJECT - Out-of-range values lose their *Flag* membership and revert to :class:`int`. + Out-of-range values lose their :class:`Flag` membership and revert to :class:`int`. >>> from enum import Flag, EJECT, auto >>> class EjectFlag(Flag, boundary=EJECT): @@ -742,7 +761,7 @@ Data Types .. attribute:: KEEP - Out-of-range values are kept, and the *Flag* membership is kept. + Out-of-range values are kept, and the :class:`Flag` membership is kept. This is the default for :class:`IntFlag`:: >>> from enum import Flag, KEEP, auto @@ -764,37 +783,41 @@ Supported ``__dunder__`` names :attr:`~EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` items. It is only available on the class. -:meth:`~object.__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`!_value_` appropriately. Once -all the members are created it is no longer used. +:meth:`~object.__new__`, if specified, must create and return the enum members; +it is also a very good idea to set the member's :attr:`!_value_` appropriately. +Once all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` - -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, - that will not be transformed into members, and will be removed from the final - class -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used to get an appropriate value for an enum - member; may be overridden - - .. note:: - - For standard :class:`Enum` classes the next value chosen is the last value seen - incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two, regardless of the last value seen. +- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an + existing member. +- :attr:`~Enum._name_` -- name of the member +- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` +- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; + may be overridden +- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a + :class:`str`, that will not be transformed into members, and will be removed + from the final class +- :attr:`~Enum._order_` -- used in Python 2/3 code to ensure member order is + consistent (class attribute, removed during class creation) +- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for + an enum member; may be overridden + + .. note:: + + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. + + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two. .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` --------------- @@ -804,20 +827,20 @@ Utilities and Decorators .. class:: auto *auto* can be used in place of a value. If used, the *Enum* machinery will - call an *Enum*'s :meth:`~Enum._generate_next_value_` to get an appropriate value. - For *Enum* and *IntEnum* that appropriate value will be the last value plus - one; for *Flag* and *IntFlag* it will be the first power-of-two greater - than the highest value; for *StrEnum* it will be the lower-cased version of + call an :class:`Enum`'s :meth:`~Enum._generate_next_value_` to get an appropriate value. + For :class:`Enum` and :class:`IntEnum` that appropriate value will be the last value plus + one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater + than the highest value; for :class:`StrEnum` it will be the lower-cased version of the member's name. Care must be taken if mixing *auto()* with manually specified values. *auto* instances are only resolved when at the top level of an assignment: - * ``FIRST = auto()`` will work (auto() is replaced with ``1``); - * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is - used to create the ``SECOND`` enum member; - * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to - create the ``THREE`` enum member) + * ``FIRST = auto()`` will work (auto() is replaced with ``1``); + * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is + used to create the ``SECOND`` enum member; + * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to + create the ``THREE`` enum member) .. versionchanged:: 3.11.1 diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 5122c69697ef91..283e8b013265d9 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -511,6 +511,13 @@ defined by the module. The specific list of defined symbols is available as Operation not supported on transport endpoint +.. data:: ENOTSUP + + Operation not supported + + .. versionadded:: 3.2 + + .. data:: EPFNOSUPPORT Protocol family not supported @@ -666,3 +673,24 @@ defined by the module. The specific list of defined symbols is available as .. availability:: WASI, FreeBSD .. versionadded:: 3.11.1 + + +.. data:: ECANCELED + + Operation canceled + + .. versionadded:: 3.2 + + +.. data:: EOWNERDEAD + + Owner died + + .. versionadded:: 3.2 + + +.. data:: ENOTRECOVERABLE + + State not recoverable + + .. versionadded:: 3.2 diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 4c84e5f855431a..b67215b8b3a362 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -220,10 +220,16 @@ The following exceptions are the exceptions that are usually raised. load a module. Also raised when the "from list" in ``from ... import`` has a name that cannot be found. - The :attr:`name` and :attr:`path` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the module - that was attempted to be imported and the path to any file which triggered - the exception, respectively. + The optional *name* and *path* keyword-only arguments + set the corresponding attributes: + + .. attribute:: name + + The name of the module that was attempted to be imported. + + .. attribute:: path + + The path to any file which triggered the exception. .. versionchanged:: 3.3 Added the :attr:`name` and :attr:`path` attributes. @@ -423,9 +429,11 @@ The following exceptions are the exceptions that are usually raised. :meth:`~iterator.__next__` method to signal that there are no further items produced by the iterator. - The exception object has a single attribute :attr:`value`, which is - given as an argument when constructing the exception, and defaults - to :const:`None`. + .. attribute:: StopIteration.value + + The exception object has a single attribute :attr:`!value`, which is + given as an argument when constructing the exception, and defaults + to :const:`None`. When a :term:`generator` or :term:`coroutine` function returns, a new :exc:`StopIteration` instance is @@ -450,7 +458,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: StopAsyncIteration - Must be raised by :meth:`__anext__` method of an + Must be raised by :meth:`~object.__anext__` method of an :term:`asynchronous iterator` object to stop the iteration. .. versionadded:: 3.5 @@ -659,8 +667,8 @@ depending on the system error code. Raised when an operation would block on an object (e.g. socket) set for non-blocking operation. - Corresponds to :c:data:`errno` :py:data:`~errno.EAGAIN`, :py:data:`~errno.EALREADY`, - :py:data:`~errno.EWOULDBLOCK` and :py:data:`~errno.EINPROGRESS`. + Corresponds to :c:data:`errno` :py:const:`~errno.EAGAIN`, :py:const:`~errno.EALREADY`, + :py:const:`~errno.EWOULDBLOCK` and :py:const:`~errno.EINPROGRESS`. In addition to those of :exc:`OSError`, :exc:`BlockingIOError` can have one more attribute: @@ -674,7 +682,7 @@ depending on the system error code. .. exception:: ChildProcessError Raised when an operation on a child process failed. - Corresponds to :c:data:`errno` :py:data:`~errno.ECHILD`. + Corresponds to :c:data:`errno` :py:const:`~errno.ECHILD`. .. exception:: ConnectionError @@ -688,40 +696,40 @@ depending on the system error code. A subclass of :exc:`ConnectionError`, raised when trying to write on a pipe while the other end has been closed, or trying to write on a socket which has been shutdown for writing. - Corresponds to :c:data:`errno` :py:data:`~errno.EPIPE` and :py:data:`~errno.ESHUTDOWN`. + Corresponds to :c:data:`errno` :py:const:`~errno.EPIPE` and :py:const:`~errno.ESHUTDOWN`. .. exception:: ConnectionAbortedError A subclass of :exc:`ConnectionError`, raised when a connection attempt is aborted by the peer. - Corresponds to :c:data:`errno` :py:data:`~errno.ECONNABORTED`. + Corresponds to :c:data:`errno` :py:const:`~errno.ECONNABORTED`. .. exception:: ConnectionRefusedError A subclass of :exc:`ConnectionError`, raised when a connection attempt is refused by the peer. - Corresponds to :c:data:`errno` :py:data:`~errno.ECONNREFUSED`. + Corresponds to :c:data:`errno` :py:const:`~errno.ECONNREFUSED`. .. exception:: ConnectionResetError A subclass of :exc:`ConnectionError`, raised when a connection is reset by the peer. - Corresponds to :c:data:`errno` :py:data:`~errno.ECONNRESET`. + Corresponds to :c:data:`errno` :py:const:`~errno.ECONNRESET`. .. exception:: FileExistsError Raised when trying to create a file or directory which already exists. - Corresponds to :c:data:`errno` :py:data:`~errno.EEXIST`. + Corresponds to :c:data:`errno` :py:const:`~errno.EEXIST`. .. exception:: FileNotFoundError Raised when a file or directory is requested but doesn't exist. - Corresponds to :c:data:`errno` :py:data:`~errno.ENOENT`. + Corresponds to :c:data:`errno` :py:const:`~errno.ENOENT`. .. exception:: InterruptedError Raised when a system call is interrupted by an incoming signal. - Corresponds to :c:data:`errno` :py:data:`~errno.EINTR`. + Corresponds to :c:data:`errno` :py:const:`~errno.EINTR`. .. versionchanged:: 3.5 Python now retries system calls when a syscall is interrupted by a @@ -732,7 +740,7 @@ depending on the system error code. Raised when a file operation (such as :func:`os.remove`) is requested on a directory. - Corresponds to :c:data:`errno` :py:data:`~errno.EISDIR`. + Corresponds to :c:data:`errno` :py:const:`~errno.EISDIR`. .. exception:: NotADirectoryError @@ -740,28 +748,28 @@ depending on the system error code. something which is not a directory. On most POSIX platforms, it may also be raised if an operation attempts to open or traverse a non-directory file as if it were a directory. - Corresponds to :c:data:`errno` :py:data:`~errno.ENOTDIR`. + Corresponds to :c:data:`errno` :py:const:`~errno.ENOTDIR`. .. exception:: PermissionError Raised when trying to run an operation without the adequate access rights - for example filesystem permissions. - Corresponds to :c:data:`errno` :py:data:`~errno.EACCES`, - :py:data:`~errno.EPERM`, and :py:data:`~errno.ENOTCAPABLE`. + Corresponds to :c:data:`errno` :py:const:`~errno.EACCES`, + :py:const:`~errno.EPERM`, and :py:const:`~errno.ENOTCAPABLE`. .. versionchanged:: 3.11.1 - WASI's :py:data:`~errno.ENOTCAPABLE` is now mapped to + WASI's :py:const:`~errno.ENOTCAPABLE` is now mapped to :exc:`PermissionError`. .. exception:: ProcessLookupError Raised when a given process doesn't exist. - Corresponds to :c:data:`errno` :py:data:`~errno.ESRCH`. + Corresponds to :c:data:`errno` :py:const:`~errno.ESRCH`. .. exception:: TimeoutError Raised when a system function timed out at the system level. - Corresponds to :c:data:`errno` :py:data:`~errno.ETIMEDOUT`. + Corresponds to :c:data:`errno` :py:const:`~errno.ETIMEDOUT`. .. versionadded:: 3.3 All the above :exc:`OSError` subclasses were added. @@ -871,6 +879,8 @@ The following exceptions are used as warning categories; see the .. versionadded:: 3.2 +.. _lib-exception-groups: + Exception groups ---------------- @@ -912,10 +922,11 @@ their subgroups based on the types of the contained exceptions. Returns an exception group that contains only the exceptions from the current group that match *condition*, or ``None`` if the result is empty. - The condition can be either a function that accepts an exception and returns - true for those that should be in the subgroup, or it can be an exception type - or a tuple of exception types, which is used to check for a match using the - same check that is used in an ``except`` clause. + The condition can be an exception type or tuple of exception types, in which + case each exception is checked for a match using the same check that is used + in an ``except`` clause. The condition can also be a callable (other than + a type object) that accepts an exception as its single argument and returns + true for the exceptions that should be in the subgroup. The nesting structure of the current exception is preserved in the result, as are the values of its :attr:`message`, :attr:`__traceback__`, @@ -926,6 +937,9 @@ their subgroups based on the types of the contained exceptions. including the top-level and any nested exception groups. If the condition is true for such an exception group, it is included in the result in full. + .. versionadded:: 3.13 + ``condition`` can be any callable which is not a type object. + .. method:: split(condition) Like :meth:`subgroup`, but returns the pair ``(match, rest)`` where ``match`` diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 997c7ea571fc03..309ad652d4af34 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -18,7 +18,7 @@ interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. For a complete description of these calls, see :manpage:`fcntl(2)` and :manpage:`ioctl(2)` Unix manual pages. -.. include:: ../includes/wasm-notavail.rst +.. availability:: Unix, not Emscripten, not WASI. All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by @@ -172,9 +172,9 @@ The module defines the following functions: which the lock starts, relative to *whence*, and *whence* is as with :func:`io.IOBase.seek`, specifically: - * :const:`0` -- relative to the start of the file (:data:`os.SEEK_SET`) - * :const:`1` -- relative to the current buffer position (:data:`os.SEEK_CUR`) - * :const:`2` -- relative to the end of the file (:data:`os.SEEK_END`) + * ``0`` -- relative to the start of the file (:const:`os.SEEK_SET`) + * ``1`` -- relative to the current buffer position (:const:`os.SEEK_CUR`) + * ``2`` -- relative to the end of the file (:const:`os.SEEK_END`) The default for *start* is 0, which means to start at the beginning of the file. The default for *len* is 0 which means to lock to the end of the file. The @@ -201,7 +201,7 @@ using the :func:`flock` call may be better. .. seealso:: Module :mod:`os` - If the locking flags :data:`~os.O_SHLOCK` and :data:`~os.O_EXLOCK` are + If the locking flags :const:`~os.O_SHLOCK` and :const:`~os.O_EXLOCK` are present in the :mod:`os` module (on BSD only), the :func:`os.open` function provides an alternative to the :func:`lockf` and :func:`flock` functions. diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index 83e9e14ddcacd8..dfe4b7c59fd578 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -74,7 +74,7 @@ The :class:`dircmp` class Construct a new directory comparison object, to compare the directories *a* and *b*. *ignore* is a list of names to ignore, and defaults to - :attr:`filecmp.DEFAULT_IGNORES`. *hide* is a list of names to hide, and + :const:`filecmp.DEFAULT_IGNORES`. *hide* is a list of names to hide, and defaults to ``[os.curdir, os.pardir]``. The :class:`dircmp` class compares files by doing *shallow* comparisons @@ -100,7 +100,7 @@ The :class:`dircmp` class used to get various bits of information about the directory trees being compared. - Note that via :meth:`__getattr__` hooks, all attributes are computed lazily, + Note that via :meth:`~object.__getattr__` hooks, all attributes are computed lazily, so there is no speed penalty if only those attributes which are lightweight to compute are used. diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 4bc868759f2025..f93e9a58791eeb 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -177,7 +177,7 @@ available for subclassing as well: The keyword-only parameter *encoding* and *errors* are added. .. versionchanged:: 3.11 - The ``'rU'`` and ``'U'`` modes and the :meth:`__getitem__` method have + The ``'rU'`` and ``'U'`` modes and the :meth:`!__getitem__` method have been removed. diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index fe2e8ab655edf8..509c63686f5a7f 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -25,7 +25,7 @@ another rational number, or from a string. The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value ``numerator/denominator``. If *denominator* is :const:`0`, it + with value ``numerator/denominator``. If *denominator* is ``0``, it raises a :exc:`ZeroDivisionError`. The second version requires that *other_fraction* is an instance of :class:`numbers.Rational` and returns a :class:`Fraction` instance with the same value. The next two versions accept diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 47054812f9f514..d1fe6414ea020c 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -105,13 +105,7 @@ The module defines the following items: .. versionchanged:: 3.4 The class now supports hostname check with :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. + :const:`ssl.HAS_SNI`). .. versionchanged:: 3.9 If the *timeout* parameter is set to be zero, it will raise a @@ -120,7 +114,7 @@ The module defines the following items: Latin-1 to UTF-8 to follow :rfc:`2640`. .. versionchanged:: 3.12 - The deprecated *keyfile* and *certfile* parameters have been removed. + The deprecated *keyfile* and *certfile* parameters have been removed. Here's a sample session using the :class:`FTP_TLS` class:: @@ -437,7 +431,7 @@ FTP_TLS Objects .. attribute:: FTP_TLS.ssl_version - The SSL version to use (defaults to :attr:`ssl.PROTOCOL_SSLv23`). + The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). .. method:: FTP_TLS.auth() @@ -447,7 +441,7 @@ FTP_TLS Objects .. versionchanged:: 3.4 The method now supports hostname check with :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). + :const:`ssl.HAS_SNI`). .. method:: FTP_TLS.ccc() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3d2bb8efc95d8e..c731b6fd333275 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -14,8 +14,8 @@ are always available. They are listed here in alphabetical order. | | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | | | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | | | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`any` | | | | | | :func:`round` | -| | :func:`anext` | | **F** | | **M** | | | +| | :func:`anext` | | | | | | :func:`round` | +| | :func:`any` | | **F** | | **M** | | | | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | @@ -57,7 +57,8 @@ are always available. They are listed here in alphabetical order. .. function:: abs(x) Return the absolute value of a number. The argument may be an - integer, a floating point number, or an object implementing :meth:`__abs__`. + integer, a floating point number, or an object implementing + :meth:`~object.__abs__`. If the argument is a complex number, its magnitude is returned. @@ -122,7 +123,7 @@ are always available. They are listed here in alphabetical order. Convert an integer number to a binary string prefixed with "0b". The result is a valid Python expression. If *x* is not a Python :class:`int` object, it - has to define an :meth:`__index__` method that returns an integer. Some + has to define an :meth:`~object.__index__` method that returns an integer. Some examples: >>> bin(3) @@ -235,7 +236,7 @@ are always available. They are listed here in alphabetical order. :const:`False` if not. If this returns ``True``, it is still possible that a call fails, but if it is ``False``, calling *object* will never succeed. Note that classes are callable (calling a class returns a new instance); - instances are callable if their class has a :meth:`__call__` method. + instances are callable if their class has a :meth:`~object.__call__` method. .. versionadded:: 3.2 This function was first removed in Python 3.0 and then brought back @@ -285,7 +286,7 @@ are always available. They are listed here in alphabetical order. ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``) and have a new ``__wrapped__`` attribute. - .. versionchanged:: 3.11 + .. deprecated-removed:: 3.11 3.13 Class methods can no longer wrap other :term:`descriptors ` such as :func:`property`. @@ -383,9 +384,9 @@ are always available. They are listed here in alphabetical order. ``0j``. For a general Python object ``x``, ``complex(x)`` delegates to - ``x.__complex__()``. If ``__complex__()`` is not defined then it falls back - to :meth:`__float__`. If ``__float__()`` is not defined then it falls back - to :meth:`__index__`. + ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + to :meth:`~object.__index__`. .. note:: @@ -400,8 +401,8 @@ are always available. They are listed here in alphabetical order. Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.8 - Falls back to :meth:`__index__` if :meth:`__complex__` and - :meth:`__float__` are not defined. + Falls back to :meth:`~object.__index__` if :meth:`~object.__complex__` and + :meth:`~object.__float__` are not defined. .. function:: delattr(object, name) @@ -432,15 +433,18 @@ are always available. They are listed here in alphabetical order. Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object. - If the object has a method named :meth:`__dir__`, this method will be called and + If the object has a method named :meth:`~object.__dir__`, + this method will be called and must return the list of attributes. This allows objects that implement a custom - :func:`__getattr__` or :func:`__getattribute__` function to customize the way + :func:`~object.__getattr__` or :func:`~object.__getattribute__` function + to customize the way :func:`dir` reports their attributes. - If the object does not provide :meth:`__dir__`, the function tries its best to - gather information from the object's :attr:`~object.__dict__` attribute, if defined, and + If the object does not provide :meth:`~object.__dir__`, + the function tries its best to gather information from the object's + :attr:`~object.__dict__` attribute, if defined, and from its type object. The resulting list is not necessarily complete and may - be inaccurate when the object has a custom :func:`__getattr__`. + be inaccurate when the object has a custom :func:`~object.__getattr__`. The default :func:`dir` mechanism behaves differently with different types of objects, as it attempts to produce the most relevant, rather than complete, @@ -664,7 +668,7 @@ are always available. They are listed here in alphabetical order. sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" - digitpart: `digit` (["_"] `digit`)* + digitpart: `!digit` (["_"] `!digit`)* number: [`digitpart`] "." `digitpart` | `digitpart` ["."] exponent: ("e" | "E") ["+" | "-"] `digitpart` floatnumber: number [`exponent`] @@ -681,8 +685,8 @@ are always available. They are listed here in alphabetical order. float, an :exc:`OverflowError` will be raised. For a general Python object ``x``, ``float(x)`` delegates to - ``x.__float__()``. If ``__float__()`` is not defined then it falls back - to :meth:`__index__`. + ``x.__float__()``. If :meth:`~object.__float__` is not defined then it falls back + to :meth:`~object.__index__`. If no argument is given, ``0.0`` is returned. @@ -708,7 +712,7 @@ are always available. They are listed here in alphabetical order. *x* is now a positional-only parameter. .. versionchanged:: 3.8 - Falls back to :meth:`__index__` if :meth:`__float__` is not defined. + Falls back to :meth:`~object.__index__` if :meth:`~object.__float__` is not defined. .. index:: @@ -727,8 +731,8 @@ are always available. They are listed here in alphabetical order. A call to ``format(value, format_spec)`` is translated to ``type(value).__format__(value, format_spec)`` which bypasses the instance - dictionary when searching for the value's :meth:`__format__` method. A - :exc:`TypeError` exception is raised if the method search reaches + dictionary when searching for the value's :meth:`~object.__format__` method. + A :exc:`TypeError` exception is raised if the method search reaches :mod:`object` and the *format_spec* is non-empty, or if either the *format_spec* or the return value are not strings. @@ -792,9 +796,9 @@ are always available. They are listed here in alphabetical order. .. note:: - For objects with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`~object.__hash__` methods, + note that :func:`hash` truncates the return value based on the bit width of the host machine. - See :meth:`__hash__` for details. .. function:: help() help(request) @@ -822,7 +826,7 @@ are always available. They are listed here in alphabetical order. Convert an integer number to a lowercase hexadecimal string prefixed with "0x". If *x* is not a Python :class:`int` object, it has to define an - :meth:`__index__` method that returns an integer. Some examples: + :meth:`~object.__index__` method that returns an integer. Some examples: >>> hex(255) '0xff' @@ -893,9 +897,9 @@ are always available. They are listed here in alphabetical order. int(x, base=10) Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* defines :meth:`__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__index__`, - it returns ``x.__index__()``. If *x* defines :meth:`__trunc__`, + ``0`` if no arguments are given. If *x* defines :meth:`~object.__int__`, + ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`~object.__index__`, + it returns ``x.__index__()``. If *x* defines :meth:`~object.__trunc__`, it returns ``x.__trunc__()``. For floating point numbers, this truncates towards zero. @@ -932,10 +936,10 @@ are always available. They are listed here in alphabetical order. *x* is now a positional-only parameter. .. versionchanged:: 3.8 - Falls back to :meth:`__index__` if :meth:`__int__` is not defined. + Falls back to :meth:`~object.__index__` if :meth:`~object.__int__` is not defined. .. versionchanged:: 3.11 - The delegation to :meth:`__trunc__` is deprecated. + The delegation to :meth:`~object.__trunc__` is deprecated. .. versionchanged:: 3.11 :class:`int` string inputs and string representations can be limited to @@ -982,8 +986,9 @@ are always available. They are listed here in alphabetical order. Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a second argument, *object* must be a collection object which supports the - :term:`iterable` protocol (the :meth:`__iter__` method), or it must support - the sequence protocol (the :meth:`__getitem__` method with integer arguments + :term:`iterable` protocol (the :meth:`~object.__iter__` method), + or it must support + the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, then *object* must be a callable object. The iterator created in this case @@ -1138,7 +1143,7 @@ are always available. They are listed here in alphabetical order. Convert an integer number to an octal string prefixed with "0o". The result is a valid Python expression. If *x* is not a Python :class:`int` object, it - has to define an :meth:`__index__` method that returns an integer. For + has to define an :meth:`~object.__index__` method that returns an integer. For example: >>> oct(8) @@ -1158,8 +1163,8 @@ are always available. They are listed here in alphabetical order. See also :func:`format` for more information. - .. index:: - single: file object; open() built-in function +.. index:: + single: file object; open() built-in function .. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None) @@ -1221,7 +1226,7 @@ are always available. They are listed here in alphabetical order. *buffering* is an optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line - buffering (only usable in text mode), and an integer > 1 to indicate the size + buffering (only usable when writing in text mode), and an integer > 1 to indicate the size in bytes of a fixed-size chunk buffer. Note that specifying a buffer size this way applies for binary buffered I/O, but ``TextIOWrapper`` (i.e., files opened with ``mode='r+'``) would have another buffering. To disable buffering in @@ -1231,7 +1236,7 @@ are always available. They are listed here in alphabetical order. * Binary files are buffered in fixed-size chunks; the size of the buffer is chosen using a heuristic trying to determine the underlying device's "block - size" and falling back on :attr:`io.DEFAULT_BUFFER_SIZE`. On many systems, + size" and falling back on :const:`io.DEFAULT_BUFFER_SIZE`. On many systems, the buffer will typically be 4096 or 8192 bytes long. * "Interactive" text files (files for which :meth:`~io.IOBase.isatty` @@ -1271,7 +1276,7 @@ are always available. They are listed here in alphabetical order. * ``'xmlcharrefreplace'`` is only supported when writing to a file. Characters not supported by the encoding are replaced with the - appropriate XML character reference ``&#nnn;``. + appropriate XML character reference :samp:`&#{nnn};`. * ``'backslashreplace'`` replaces malformed data by Python's backslashed escape sequences. @@ -1360,28 +1365,28 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.3 - * The *opener* parameter was added. - * The ``'x'`` mode was added. - * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. - * :exc:`FileExistsError` is now raised if the file opened in exclusive - creation mode (``'x'``) already exists. + * The *opener* parameter was added. + * The ``'x'`` mode was added. + * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. + * :exc:`FileExistsError` is now raised if the file opened in exclusive + creation mode (``'x'``) already exists. .. versionchanged:: 3.4 - * The file is now non-inheritable. + * The file is now non-inheritable. .. versionchanged:: 3.5 - * If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - * The ``'namereplace'`` error handler was added. + * If the system call is interrupted and the signal handler does not raise an + exception, the function now retries the system call instead of raising an + :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + * The ``'namereplace'`` error handler was added. .. versionchanged:: 3.6 - * Support added to accept objects implementing :class:`os.PathLike`. - * On Windows, opening a console buffer may return a subclass of - :class:`io.RawIOBase` other than :class:`io.FileIO`. + * Support added to accept objects implementing :class:`os.PathLike`. + * On Windows, opening a console buffer may return a subclass of + :class:`io.RawIOBase` other than :class:`io.FileIO`. .. versionchanged:: 3.11 The ``'U'`` mode has been removed. @@ -1500,38 +1505,44 @@ are always available. They are listed here in alphabetical order. """Get the current voltage.""" return self._voltage - The ``@property`` decorator turns the :meth:`voltage` method into a "getter" + The ``@property`` decorator turns the :meth:`!voltage` method into a "getter" for a read-only attribute with the same name, and it sets the docstring for *voltage* to "Get the current voltage." - A property object has :attr:`~property.getter`, :attr:`~property.setter`, - and :attr:`~property.deleter` methods usable as decorators that create a - copy of the property with the corresponding accessor function set to the - decorated function. This is best explained with an example:: + .. decorator:: property.getter + .. decorator:: property.setter + .. decorator:: property.deleter - class C: - def __init__(self): - self._x = None + A property object has ``getter``, ``setter``, + and ``deleter`` methods usable as decorators that create a + copy of the property with the corresponding accessor function set to the + decorated function. This is best explained with an example: - @property - def x(self): - """I'm the 'x' property.""" - return self._x + .. testcode:: - @x.setter - def x(self, value): - self._x = value + class C: + def __init__(self): + self._x = None - @x.deleter - def x(self): - del self._x + @property + def x(self): + """I'm the 'x' property.""" + return self._x + + @x.setter + def x(self, value): + self._x = value + + @x.deleter + def x(self): + del self._x - This code is exactly equivalent to the first example. Be sure to give the - additional functions the same name as the original property (``x`` in this - case.) + This code is exactly equivalent to the first example. Be sure to give the + additional functions the same name as the original property (``x`` in this + case.) - The returned property object also has the attributes ``fget``, ``fset``, and - ``fdel`` corresponding to the constructor arguments. + The returned property object also has the attributes ``fget``, ``fset``, and + ``fdel`` corresponding to the constructor arguments. .. versionchanged:: 3.5 The docstrings of property objects are now writeable. @@ -1554,7 +1565,8 @@ are always available. They are listed here in alphabetical order. representation is a string enclosed in angle brackets that contains the name of the type of the object together with additional information often including the name and address of the object. A class can control what this - function returns for its instances by defining a :meth:`__repr__` method. + function returns for its instances + by defining a :meth:`~object.__repr__` method. If :func:`sys.displayhook` is not accessible, this function will raise :exc:`RuntimeError`. @@ -1562,9 +1574,9 @@ are always available. They are listed here in alphabetical order. .. function:: reversed(seq) Return a reverse :term:`iterator`. *seq* must be an object which has - a :meth:`__reversed__` method or supports the sequence protocol (the - :meth:`__len__` method and the :meth:`__getitem__` method with integer - arguments starting at ``0``). + a :meth:`~object.__reversed__` method or supports the sequence protocol (the + :meth:`~object.__len__` method and the :meth:`~object.__getitem__` method + with integer arguments starting at ``0``). .. function:: round(number, ndigits=None) @@ -1631,17 +1643,25 @@ are always available. They are listed here in alphabetical order. .. class:: slice(stop) - slice(start, stop, step=1) + slice(start, stop, step=None) Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to - ``None``. Slice objects have read-only data attributes :attr:`~slice.start`, - :attr:`~slice.stop`, and :attr:`~slice.step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. + ``None``. + + .. attribute:: slice.start + .. attribute:: slice.stop + .. attribute:: slice.step + + Slice objects have read-only data attributes :attr:`!start`, + :attr:`!stop`, and :attr:`!step` which merely return the argument + values (or their default). They have no other explicit functionality; + however, they are used by NumPy and other third-party packages. + Slice objects are also generated when extended indexing syntax is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an iterator. + :func:`itertools.islice` for an alternate version that returns an + :term:`iterator`. .. versionchanged:: 3.12 Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, @@ -1752,7 +1772,7 @@ are always available. They are listed here in alphabetical order. The *start* parameter can be specified as a keyword argument. .. versionchanged:: 3.12 Summation of floats switched to an algorithm - that gives higher accuracy on most builds. + that gives higher accuracy and better commutativity on most builds. .. class:: super() @@ -1808,7 +1828,8 @@ are always available. They are listed here in alphabetical order. Note that :func:`super` is implemented as part of the binding process for explicit dotted attribute lookups such as ``super().__getitem__(name)``. - It does so by implementing its own :meth:`__getattribute__` method for searching + It does so by implementing its own :meth:`~object.__getattribute__` method + for searching classes in a predictable order that supports cooperative multiple inheritance. Accordingly, :func:`super` is undefined for implicit lookups using statements or operators such as ``super()[name]``. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3f22e3ff7dd119..9749328ed6e929 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -222,8 +222,9 @@ The :mod:`functools` module defines the following functions: In general, the LRU cache should only be used when you want to reuse previously computed values. Accordingly, it doesn't make sense to cache - functions with side-effects, functions that need to create distinct mutable - objects on each call, or impure functions such as time() or random(). + functions with side-effects, functions that need to create + distinct mutable objects on each call (such as generators and async functions), + or impure functions such as time() or random(). Example of an LRU cache for static web content:: @@ -398,25 +399,27 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 -.. function:: reduce(function, iterable[, initializer]) +.. function:: reduce(function, iterable[, initial], /) Apply *function* of two arguments cumulatively to the items of *iterable*, from left to right, so as to reduce the iterable to a single value. For example, ``reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])`` calculates ``((((1+2)+3)+4)+5)``. The left argument, *x*, is the accumulated value and the right argument, *y*, is - the update value from the *iterable*. If the optional *initializer* is present, + the update value from the *iterable*. If the optional *initial* is present, it is placed before the items of the iterable in the calculation, and serves as - a default when the iterable is empty. If *initializer* is not given and + a default when the iterable is empty. If *initial* is not given and *iterable* contains only one item, the first item is returned. Roughly equivalent to:: - def reduce(function, iterable, initializer=None): + initial_missing = object() + + def reduce(function, iterable, initial=initial_missing, /): it = iter(iterable) - if initializer is None: + if initial is initial_missing: value = next(it) else: - value = initializer + value = initial for element in it: value = function(value, element) return value diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 6d5c64df1a1f3f..82277aa52aee01 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -42,14 +42,17 @@ The :mod:`gc` module provides the following functions: With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The number of - unreachable objects found is returned. + :exc:`ValueError` is raised if the generation number is invalid. The sum of + collected objects and uncollectable objects is returned. The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the particular implementation, in particular :class:`float`. + The effect of calling ``gc.collect()`` while the interpreter is already + performing a collection is undefined. + .. function:: set_debug(flags) @@ -257,7 +260,7 @@ values but should not rebind them): .. versionchanged:: 3.4 Following :pep:`442`, objects with a :meth:`~object.__del__` method don't end - up in :attr:`gc.garbage` anymore. + up in :data:`gc.garbage` anymore. .. data:: callbacks diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index 336deab28cb8a4..ada68b240143e8 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -7,18 +7,23 @@ **Source code:** :source:`Lib/getopt.py` +.. deprecated:: 3.13 + The :mod:`getopt` module is :term:`soft deprecated` and will not be + developed further; development will continue with the :mod:`argparse` + module. + .. note:: The :mod:`getopt` module is a parser for command line options whose API is - designed to be familiar to users of the C :c:func:`getopt` function. Users who - are unfamiliar with the C :c:func:`getopt` function or who would like to write + designed to be familiar to users of the C :c:func:`!getopt` function. Users who + are unfamiliar with the C :c:func:`!getopt` function or who would like to write less code and get better help and error messages should consider using the :mod:`argparse` module instead. -------------- This module helps scripts to parse the command line arguments in ``sys.argv``. -It supports the same conventions as the Unix :c:func:`getopt` function (including +It supports the same conventions as the Unix :c:func:`!getopt` function (including the special meanings of arguments of the form '``-``' and '``--``'). Long options similar to those supported by GNU software may be used as well via an optional third argument. @@ -33,11 +38,11 @@ exception: be parsed, without the leading reference to the running program. Typically, this means ``sys.argv[1:]``. *shortopts* is the string of option letters that the script wants to recognize, with options that require an argument followed by a - colon (``':'``; i.e., the same format that Unix :c:func:`getopt` uses). + colon (``':'``; i.e., the same format that Unix :c:func:`!getopt` uses). .. note:: - Unlike GNU :c:func:`getopt`, after a non-option argument, all further + Unlike GNU :c:func:`!getopt`, after a non-option argument, all further arguments are considered also non-options. This is similar to the way non-GNU Unix systems work. @@ -71,7 +76,7 @@ exception: non-option argument is encountered. If the first character of the option string is ``'+'``, or if the environment - variable :envvar:`POSIXLY_CORRECT` is set, then option processing stops as + variable :envvar:`!POSIXLY_CORRECT` is set, then option processing stops as soon as a non-option argument is encountered. @@ -81,9 +86,9 @@ exception: an option requiring an argument is given none. The argument to the exception is a string indicating the cause of the error. For long options, an argument given to an option which does not require one will also cause this exception to be - raised. The attributes :attr:`msg` and :attr:`opt` give the error message and + raised. The attributes :attr:`!msg` and :attr:`!opt` give the error message and related option; if there is no specific option to which the exception relates, - :attr:`opt` is an empty string. + :attr:`!opt` is an empty string. .. XXX deprecated? .. exception:: error diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index d5bbe67fb30a62..54c84d45a59856 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -43,10 +43,13 @@ The :mod:`getpass` module provides two functions: Return the "login name" of the user. This function checks the environment variables :envvar:`LOGNAME`, - :envvar:`USER`, :envvar:`LNAME` and :envvar:`USERNAME`, in order, and + :envvar:`USER`, :envvar:`!LNAME` and :envvar:`USERNAME`, in order, and returns the value of the first one which is set to a non-empty string. If none are set, the login name from the password database is returned on - systems which support the :mod:`pwd` module, otherwise, an exception is - raised. + systems which support the :mod:`pwd` module, otherwise, an :exc:`OSError` + is raised. In general, this function should be preferred over :func:`os.getlogin()`. + + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` were raised. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 747f8703b750ec..41beac3e0c7396 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -58,7 +58,7 @@ class-based API instead. Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as - :func:`_` in the local namespace (see examples below). + :func:`!_` in the local namespace (see examples below). .. function:: dgettext(domain, message) @@ -98,7 +98,7 @@ class-based API instead. .. versionadded:: 3.8 -Note that GNU :program:`gettext` also defines a :func:`dcgettext` method, but +Note that GNU :program:`gettext` also defines a :func:`!dcgettext` method, but this was deemed not useful and so it is currently unimplemented. Here's an example of typical usage for this API:: @@ -119,7 +119,7 @@ greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format files, and has methods for returning strings. Instances of this class can also -install themselves in the built-in namespace as the function :func:`_`. +install themselves in the built-in namespace as the function :func:`!_`. .. function:: find(domain, localedir=None, languages=None, all=False) @@ -150,15 +150,12 @@ install themselves in the built-in namespace as the function :func:`_`. .. function:: translation(domain, localedir=None, languages=None, class_=None, fallback=False) - Return a :class:`*Translations` instance based on the *domain*, *localedir*, + Return a ``*Translations`` instance based on the *domain*, *localedir*, and *languages*, which are first passed to :func:`find` to get a list of the associated :file:`.mo` file paths. Instances with identical :file:`.mo` file names are cached. The actual class instantiated is *class_* if provided, otherwise :class:`GNUTranslations`. The class's constructor must - take a single :term:`file object` argument. If provided, *codeset* will change - the charset used to encode translated strings in the - :meth:`~NullTranslations.lgettext` and :meth:`~NullTranslations.lngettext` - methods. + take a single :term:`file object` argument. If multiple files are found, later files are used as fallbacks for earlier ones. To allow setting the fallback, :func:`copy.copy` is used to clone each @@ -170,26 +167,26 @@ install themselves in the built-in namespace as the function :func:`_`. :class:`NullTranslations` instance if *fallback* is true. .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. + :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. .. versionchanged:: 3.11 *codeset* parameter is removed. .. function:: install(domain, localedir=None, *, names=None) - This installs the function :func:`_` in Python's builtins namespace, based on + This installs the function :func:`!_` in Python's builtins namespace, based on *domain* and *localedir* which are passed to the function :func:`translation`. For the *names* parameter, please see the description of the translation object's :meth:`~NullTranslations.install` method. As seen below, you usually mark the strings in your application that are - candidates for translation, by wrapping them in a call to the :func:`_` + candidates for translation, by wrapping them in a call to the :func:`!_` function, like this:: print(_('This string will be translated.')) - For convenience, you want the :func:`_` function to be installed in Python's + For convenience, you want the :func:`!_` function to be installed in Python's builtins namespace, so it is easily accessible in all modules of your application. @@ -260,7 +257,7 @@ are the methods of :class:`!NullTranslations`: .. method:: info() - Return the "protected" :attr:`_info` variable, a dictionary containing + Return a dictionary containing the metadata found in the message catalog file. @@ -276,20 +273,20 @@ are the methods of :class:`!NullTranslations`: If the *names* parameter is given, it must be a sequence containing the names of functions you want to install in the builtins namespace in - addition to :func:`_`. Supported names are ``'gettext'``, ``'ngettext'``, - ``'pgettext'``, ``'npgettext'``, ``'lgettext'``, and ``'lngettext'``. + addition to :func:`!_`. Supported names are ``'gettext'``, ``'ngettext'``, + ``'pgettext'``, and ``'npgettext'``. Note that this is only one way, albeit the most convenient way, to make - the :func:`_` function available to your application. Because it affects + the :func:`!_` function available to your application. Because it affects the entire application globally, and specifically the built-in namespace, - localized modules should never install :func:`_`. Instead, they should use - this code to make :func:`_` available to their module:: + localized modules should never install :func:`!_`. Instead, they should use + this code to make :func:`!_` available to their module:: import gettext t = gettext.translation('mymodule', ...) _ = t.gettext - This puts :func:`_` only in the module's global namespace and so only + This puts :func:`!_` only in the module's global namespace and so only affects calls within this module. .. versionchanged:: 3.8 @@ -299,9 +296,9 @@ are the methods of :class:`!NullTranslations`: The :class:`GNUTranslations` class ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The :mod:`gettext` module provides one additional class derived from +The :mod:`!gettext` module provides one additional class derived from :class:`NullTranslations`: :class:`GNUTranslations`. This class overrides -:meth:`_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files +:meth:`!_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files in both big-endian and little-endian format. :class:`GNUTranslations` parses optional metadata out of the translation @@ -309,16 +306,16 @@ catalog. It is convention with GNU :program:`gettext` to include metadata as the translation for the empty string. This metadata is in :rfc:`822`\ -style ``key: value`` pairs, and should contain the ``Project-Id-Version`` key. If the key ``Content-Type`` is found, then the ``charset`` property is used to -initialize the "protected" :attr:`_charset` instance variable, defaulting to +initialize the "protected" :attr:`!_charset` instance variable, defaulting to ``None`` if not found. If the charset encoding is specified, then all message ids and message strings read from the catalog are converted to Unicode using this encoding, else ASCII is assumed. -Since message ids are read as Unicode strings too, all :meth:`*gettext` methods +Since message ids are read as Unicode strings too, all ``*gettext()`` methods will assume message ids as Unicode strings, not byte strings. The entire set of key/value pairs are placed into a dictionary and set as the -"protected" :attr:`_info` instance variable. +"protected" :attr:`!_info` instance variable. If the :file:`.mo` file's magic number is invalid, the major version number is unexpected, or if other problems occur while reading the file, instantiating a @@ -404,13 +401,14 @@ version has a slightly different API. Its documented usage was:: _ = cat.gettext print(_('hello world')) -For compatibility with this older module, the function :func:`Catalog` is an +For compatibility with this older module, the function :func:`!Catalog` is an alias for the :func:`translation` function described above. One difference between this module and Henstridge's: his catalog objects supported access through a mapping API, but this appears to be unused and so is not currently supported. +.. _i18n-howto: Internationalizing your programs and modules -------------------------------------------- @@ -431,7 +429,7 @@ take the following steps: In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping -it in ``_('...')`` --- that is, a call to the function :func:`_`. For example:: +it in ``_('...')`` --- that is, a call to the function :func:`_ `. For example:: filename = 'mylog.txt' message = _('writing a log message') @@ -503,7 +501,7 @@ module:: Localizing your application ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you are localizing your application, you can install the :func:`_` function +If you are localizing your application, you can install the :func:`!_` function globally into the built-in namespace, usually in the main driver file of your application. This will let all your application-specific files just use ``_('...')`` without having to explicitly install it in each file. @@ -580,13 +578,13 @@ Here is one way you can handle this situation:: for a in animals: print(_(a)) -This works because the dummy definition of :func:`_` simply returns the string +This works because the dummy definition of :func:`!_` simply returns the string unchanged. And this dummy definition will temporarily override any definition -of :func:`_` in the built-in namespace (until the :keyword:`del` command). Take -care, though if you have a previous definition of :func:`_` in the local +of :func:`!_` in the built-in namespace (until the :keyword:`del` command). Take +care, though if you have a previous definition of :func:`!_` in the local namespace. -Note that the second use of :func:`_` will not identify "a" as being +Note that the second use of :func:`!_` will not identify "a" as being translatable to the :program:`gettext` program, because the parameter is not a string literal. @@ -605,13 +603,13 @@ Another way to handle this is with the following example:: print(_(a)) In this case, you are marking translatable strings with the function -:func:`N_`, which won't conflict with any definition of :func:`_`. +:func:`!N_`, which won't conflict with any definition of :func:`!_`. However, you will need to teach your message extraction program to -look for translatable strings marked with :func:`N_`. :program:`xgettext`, +look for translatable strings marked with :func:`!N_`. :program:`xgettext`, :program:`pygettext`, ``pybabel extract``, and :program:`xpot` all support this through the use of the :option:`!-k` command-line switch. -The choice of :func:`N_` here is totally arbitrary; it could have just -as easily been :func:`MarkThisStringForTranslation`. +The choice of :func:`!N_` here is totally arbitrary; it could have just +as easily been :func:`!MarkThisStringForTranslation`. Acknowledgements @@ -638,9 +636,9 @@ implementations, and valuable experience to the creation of this module: .. rubric:: Footnotes -.. [#] The default locale directory is system dependent; for example, on RedHat Linux +.. [#] The default locale directory is system dependent; for example, on Red Hat Linux it is :file:`/usr/share/locale`, but on Solaris it is :file:`/usr/lib/locale`. - The :mod:`gettext` module does not try to support these system dependent + The :mod:`!gettext` module does not try to support these system dependent defaults; instead its default is :file:`{sys.base_prefix}/share/locale` (see :data:`sys.base_prefix`). For this reason, it is always best to call :func:`bindtextdomain` with an explicit absolute path at the start of your diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 0e4cfe7ebed797..6e4f72c19ff4c9 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -34,9 +34,7 @@ unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. - -.. seealso:: - The :mod:`pathlib` module offers high-level path objects. +The :mod:`glob` module defines the following functions: .. function:: glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ @@ -117,7 +115,48 @@ For example, ``'[?]'`` matches the character ``'?'``. .. versionadded:: 3.4 -For example, consider a directory containing the following files: +.. function:: translate(pathname, *, recursive=False, include_hidden=False, seps=None) + + Convert the given path specification to a regular expression for use with + :func:`re.match`. The path specification can contain shell-style wildcards. + + For example: + + >>> import glob, re + >>> + >>> regex = glob.translate('**/*.txt', recursive=True, include_hidden=True) + >>> regex + '(?s:(?:.+/)?[^/]*\\.txt)\\Z' + >>> reobj = re.compile(regex) + >>> reobj.match('foo/bar/baz.txt') + + + Path separators and segments are meaningful to this function, unlike + :func:`fnmatch.translate`. By default wildcards do not match path + separators, and ``*`` pattern segments match precisely one path segment. + + If *recursive* is true, the pattern segment "``**``" will match any number + of path segments. If "``**``" occurs in any position other than a full + pattern segment, :exc:`ValueError` is raised. + + If *include_hidden* is true, wildcards can match path segments that start + with a dot (``.``). + + A sequence of path separators may be supplied to the *seps* argument. If + not given, :data:`os.sep` and :data:`~os.altsep` (if available) are used. + + .. seealso:: + + :meth:`pathlib.PurePath.match` and :meth:`pathlib.Path.glob` methods, + which call this function to implement pattern matching and globbing. + + .. versionadded:: 3.13 + + +Examples +-------- + +Consider a directory containing the following files: :file:`1.gif`, :file:`2.txt`, :file:`card.gif` and a subdirectory :file:`sub` which contains only the file :file:`3.txt`. :func:`glob` will produce the following results. Notice how any leading components of the path are @@ -146,6 +185,7 @@ default. For example, consider a directory containing :file:`card.gif` and ['.card.gif'] .. seealso:: + The :mod:`fnmatch` module offers shell-style filename (not path) expansion. - Module :mod:`fnmatch` - Shell-style filename (not path) expansion +.. seealso:: + The :mod:`pathlib` module offers high-level path objects. diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index fe7932e7a61cb5..5414d6370b78ce 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -37,14 +37,14 @@ In the general case, the steps required to perform the sorting of a given graph are as follows: - * Create an instance of the :class:`TopologicalSorter` with an optional - initial graph. - * Add additional nodes to the graph. - * Call :meth:`~TopologicalSorter.prepare` on the graph. - * While :meth:`~TopologicalSorter.is_active` is ``True``, iterate over - the nodes returned by :meth:`~TopologicalSorter.get_ready` and - process them. Call :meth:`~TopologicalSorter.done` on each node as it - finishes processing. + * Create an instance of the :class:`TopologicalSorter` with an optional + initial graph. + * Add additional nodes to the graph. + * Call :meth:`~TopologicalSorter.prepare` on the graph. + * While :meth:`~TopologicalSorter.is_active` is ``True``, iterate over + the nodes returned by :meth:`~TopologicalSorter.get_ready` and + process them. Call :meth:`~TopologicalSorter.done` on each node as it + finishes processing. In case just an immediate sorting of the nodes in the graph is required and no parallelism is involved, the convenience method @@ -115,7 +115,7 @@ :meth:`TopologicalSorter.done` is less than the number that have been returned by :meth:`TopologicalSorter.get_ready`. - The :meth:`~TopologicalSorter.__bool__` method of this class defers to + The :meth:`~object.__bool__` method of this class defers to this function, so instead of:: if ts.is_active(): @@ -204,7 +204,7 @@ The :mod:`graphlib` module defines the following exception classes: in the working graph. If multiple cycles exist, only one undefined choice among them will be reported and included in the exception. - The detected cycle can be accessed via the second element in the :attr:`~CycleError.args` + The detected cycle can be accessed via the second element in the :attr:`~BaseException.args` attribute of the exception instance and consists in a list of nodes, such that each node is, in the graph, an immediate predecessor of the next node in the list. In the reported list, the first and the last node will be the same, to make it clear that it is cyclic. diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index 14af744e3ae8f4..274a353103b488 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -10,7 +10,7 @@ This module provides access to the Unix group database. It is available on all Unix versions. -.. include:: ../includes/wasm-notavail.rst +.. availability:: Unix, not Emscripten, not WASI. Group database entries are reported as a tuple-like object, whose attributes correspond to the members of the ``group`` structure (Attribute field below, see @@ -63,7 +63,3 @@ It defines the following items: Module :mod:`pwd` An interface to the user database, similar to this. - - Module :mod:`spwd` - An interface to the shadow password database, similar to this. - diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 06cbd2567a0bc6..50cde09fa10a9d 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -70,7 +70,7 @@ The module defines the following items: .. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) Constructor for the :class:`GzipFile` class, which simulates most of the - methods of a :term:`file object`, with the exception of the :meth:`truncate` + methods of a :term:`file object`, with the exception of the :meth:`~io.IOBase.truncate` method. At least one of *fileobj* and *filename* must be given a non-trivial value. @@ -105,7 +105,7 @@ The module defines the following items: should only be provided in compression mode. If omitted or ``None``, the current time is used. See the :attr:`mtime` attribute for more details. - Calling a :class:`GzipFile` object's :meth:`close` method does not close + Calling a :class:`GzipFile` object's :meth:`!close` method does not close *fileobj*, since you might wish to append more material after the compressed data. This also allows you to pass an :class:`io.BytesIO` object opened for writing as *fileobj*, and retrieve the resulting memory buffer using the @@ -113,7 +113,7 @@ The module defines the following items: :class:`GzipFile` supports the :class:`io.BufferedIOBase` interface, including iteration and the :keyword:`with` statement. Only the - :meth:`truncate` method isn't implemented. + :meth:`~io.IOBase.truncate` method isn't implemented. :class:`GzipFile` also provides the following method and attribute: @@ -250,6 +250,8 @@ Example of how to GZIP compress a binary string:: .. program:: gzip +.. _gzip-cli: + Command Line Interface ---------------------- @@ -266,23 +268,23 @@ Once executed the :mod:`gzip` module keeps the input file(s). Command line options ^^^^^^^^^^^^^^^^^^^^ -.. cmdoption:: file +.. option:: file - If *file* is not specified, read from :attr:`sys.stdin`. + If *file* is not specified, read from :data:`sys.stdin`. -.. cmdoption:: --fast +.. option:: --fast Indicates the fastest compression method (less compression). -.. cmdoption:: --best +.. option:: --best Indicates the slowest compression method (best compression). -.. cmdoption:: -d, --decompress +.. option:: -d, --decompress Decompress the given file. -.. cmdoption:: -h, --help +.. option:: -h, --help Show the help message. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 797870b9d7e260..761dd84edee299 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -11,7 +11,7 @@ .. index:: single: message digest, MD5 - single: secure hash algorithm, SHA1, SHA224, SHA256, SHA384, SHA512 + single: secure hash algorithm, SHA1, SHA2, SHA224, SHA256, SHA384, SHA512, SHA3, Shake, Blake2 .. testsetup:: @@ -22,7 +22,8 @@ This module implements a common interface to many different secure hash and message digest algorithms. Included are the FIPS secure hash algorithms SHA1, -SHA224, SHA256, SHA384, and SHA512 (defined in FIPS 180-2) as well as RSA's MD5 +SHA224, SHA256, SHA384, SHA512, (defined in `the FIPS 180-4 standard`_), +the SHA-3 series (defined in `the FIPS 202 standard`_) as well as RSA's MD5 algorithm (defined in internet :rfc:`1321`). The terms "secure hash" and "message digest" are interchangeable. Older algorithms were called message digests. The modern term is secure hash. @@ -32,11 +33,6 @@ digests. The modern term is secure hash. If you want the adler32 or crc32 hash functions, they are available in the :mod:`zlib` module. -.. warning:: - - Some algorithms have known hash collision weaknesses, refer to the "See - also" section at the end. - .. _hash-algorithms: @@ -44,38 +40,43 @@ Hash algorithms --------------- There is one constructor method named for each type of :dfn:`hash`. All return -a hash object with the same simple interface. For example: use :func:`sha256` to -create a SHA-256 hash object. You can now feed this object with :term:`bytes-like -objects ` (normally :class:`bytes`) using the :meth:`update` method. -At any point you can ask it for the :dfn:`digest` of the -concatenation of the data fed to it so far using the :meth:`digest` or -:meth:`hexdigest` methods. +a hash object with the same simple interface. For example: use :func:`sha256` +to create a SHA-256 hash object. You can now feed this object with +:term:`bytes-like objects ` (normally :class:`bytes`) using +the :meth:`update` method. At any point you can ask it for the +:dfn:`digest` of the concatenation of the data fed to it so far using the +:meth:`digest()` or :meth:`hexdigest()` methods. -.. note:: +To allow multithreading, the Python :term:`GIL` is released while computing a +hash supplied more than 2047 bytes of data at once in its constructor or +:meth:`.update` method. - For better multithreading performance, the Python :term:`GIL` is released for - data larger than 2047 bytes at object creation or on update. - -.. note:: - - Feeding string objects into :meth:`update` is not supported, as hashes work - on bytes, not on characters. .. index:: single: OpenSSL; (use in module hashlib) Constructors for hash algorithms that are always present in this module are -:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, -:func:`sha512`, :func:`blake2b`, and :func:`blake2s`. -:func:`md5` is normally available as well, though it -may be missing or blocked if you are using a rare "FIPS compliant" build of Python. -Additional algorithms may also be available depending upon the OpenSSL -library that Python uses on your platform. On most platforms the +:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`, :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, -:func:`shake_128`, :func:`shake_256` are also available. +:func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and :func:`blake2s`. +:func:`md5` is normally available as well, though it may be missing or blocked +if you are using a rare "FIPS compliant" build of Python. +These correspond to :data:`algorithms_guaranteed`. + +Additional algorithms may also be available if your Python distribution's +:mod:`hashlib` was linked against a build of OpenSSL that provides others. +Others *are not guaranteed available* on all installations and will only be +accessible by name via :func:`new`. See :data:`algorithms_available`. + +.. warning:: + + Some algorithms have known hash collision weaknesses (including MD5 and + SHA1). Refer to `Attacks on cryptographic hash algorithms`_ and the + `hashlib-seealso`_ section at the end of this document. .. versionadded:: 3.6 SHA3 (Keccak) and SHAKE constructors :func:`sha3_224`, :func:`sha3_256`, - :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256`. + :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256` + were added. .. versionadded:: 3.6 :func:`blake2b` and :func:`blake2s` were added. @@ -89,10 +90,19 @@ library that Python uses on your platform. On most platforms the that the hashing algorithm is not used in a security context, e.g. as a non-cryptographic one-way compression function. - Hashlib now uses SHA3 and SHAKE from OpenSSL 1.1.1 and newer. +.. versionchanged:: 3.9 + Hashlib now uses SHA3 and SHAKE from OpenSSL if it provides it. + +.. versionchanged:: 3.12 + For any of the MD5, SHA1, SHA2, or SHA3 algorithms that the linked + OpenSSL does not provide we fall back to a verified implementation from + the `HACL\* project`_. -For example, to obtain the digest of the byte string ``b"Nobody inspects the -spammish repetition"``:: +Usage +----- + +To obtain the digest of the byte string ``b"Nobody inspects the spammish +repetition"``:: >>> import hashlib >>> m = hashlib.sha256() @@ -108,22 +118,42 @@ More condensed: >>> hashlib.sha256(b"Nobody inspects the spammish repetition").hexdigest() '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' -.. function:: new(name[, data], *, usedforsecurity=True) +Constructors +------------ + +.. function:: new(name[, data], \*, usedforsecurity=True) Is a generic constructor that takes the string *name* of the desired algorithm as its first parameter. It also exists to allow access to the above listed hashes as well as any other algorithms that your OpenSSL - library may offer. The named constructors are much faster than :func:`new` - and should be preferred. + library may offer. -Using :func:`new` with an algorithm provided by OpenSSL: +Using :func:`new` with an algorithm name: >>> h = hashlib.new('sha256') >>> h.update(b"Nobody inspects the spammish repetition") >>> h.hexdigest() '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' -Hashlib provides the following constant attributes: + +.. function:: md5([, data], *, usedforsecurity=True) +.. function:: sha1([, data], *, usedforsecurity=True) +.. function:: sha224([, data], *, usedforsecurity=True) +.. function:: sha256([, data], *, usedforsecurity=True) +.. function:: sha384([, data], *, usedforsecurity=True) +.. function:: sha512([, data], *, usedforsecurity=True) +.. function:: sha3_224([, data], *, usedforsecurity=True) +.. function:: sha3_256([, data], *, usedforsecurity=True) +.. function:: sha3_384([, data], *, usedforsecurity=True) +.. function:: sha3_512([, data], *, usedforsecurity=True) + +Named constructors such as these are faster than passing an algorithm name to +:func:`new`. + +Attributes +---------- + +Hashlib provides the following constant module attributes: .. data:: algorithms_guaranteed @@ -144,10 +174,12 @@ Hashlib provides the following constant attributes: .. versionadded:: 3.2 +Hash Objects +------------ + The following values are provided as constant attributes of the hash objects returned by the constructors: - .. data:: hash.digest_size The size of the resulting hash in bytes. @@ -178,11 +210,6 @@ A hash object has the following methods: concatenation of all the arguments: ``m.update(a); m.update(b)`` is equivalent to ``m.update(a+b)``. - .. versionchanged:: 3.1 - The Python GIL is released to allow other threads to run while hash - updates on data larger than 2047 bytes is taking place when using hash - algorithms supplied by OpenSSL. - .. method:: hash.digest() @@ -207,6 +234,9 @@ A hash object has the following methods: SHAKE variable length digests ----------------------------- +.. function:: shake_128([, data], *, usedforsecurity=True) +.. function:: shake_256([, data], *, usedforsecurity=True) + The :func:`shake_128` and :func:`shake_256` algorithms provide variable length digests with length_in_bits//2 up to 128 or 256 bits of security. As such, their digest methods require a length. Maximum length is not limited @@ -214,7 +244,7 @@ by the SHAKE algorithm. .. method:: shake.digest(length) - Return the digest of the data passed to the :meth:`update` method so far. + Return the digest of the data passed to the :meth:`~hash.update` method so far. This is a bytes object of size *length* which may contain bytes in the whole range from 0 to 255. @@ -223,8 +253,13 @@ by the SHAKE algorithm. Like :meth:`digest` except the digest is returned as a string object of double length, containing only hexadecimal digits. This may be used to - exchange the value safely in email or other non-binary environments. + exchange the value in email or other non-binary environments. +Example use: + + >>> h = hashlib.shake_256(b'Nobody inspects the spammish repetition') + >>> h.hexdigest(20) + '44709d6fcb83d92a76dcb0b668c98e1b1d3dafe7' File hashing ------------ @@ -329,6 +364,8 @@ include a `salt `_. .. versionadded:: 3.6 +.. _hashlib-blake2: + BLAKE2 ------ @@ -472,9 +509,9 @@ Simple hashing To calculate hash of some data, you should first construct a hash object by calling the appropriate constructor function (:func:`blake2b` or -:func:`blake2s`), then update it with the data by calling :meth:`update` on the +:func:`blake2s`), then update it with the data by calling :meth:`~hash.update` on the object, and, finally, get the digest out of the object by calling -:meth:`digest` (or :meth:`hexdigest` for hex-encoded string). +:meth:`~hash.digest` (or :meth:`~hash.hexdigest` for hex-encoded string). >>> from hashlib import blake2b >>> h = blake2b() @@ -768,13 +805,19 @@ Domain Dedication 1.0 Universal: .. _BLAKE2: https://www.blake2.net .. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code .. _BLAKE: https://web.archive.org/web/20200918190133/https://131002.net/blake/ -.. _SHA-3: https://en.wikipedia.org/wiki/NIST_hash_function_competition +.. _SHA-3: https://en.wikipedia.org/wiki/Secure_Hash_Algorithms .. _ChaCha: https://cr.yp.to/chacha.html .. _pyblake2: https://pythonhosted.org/pyblake2/ .. _NIST-SP-800-132: https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-132.pdf .. _stackexchange pbkdf2 iterations question: https://security.stackexchange.com/questions/3959/recommended-of-iterations-when-using-pbkdf2-sha256/ +.. _Attacks on cryptographic hash algorithms: https://en.wikipedia.org/wiki/Cryptographic_hash_function#Attacks_on_cryptographic_hash_algorithms +.. _the FIPS 180-4 standard: https://csrc.nist.gov/publications/detail/fips/180/4/final +.. _the FIPS 202 standard: https://csrc.nist.gov/publications/detail/fips/202/final +.. _HACL\* project: https://github.com/hacl-star/hacl-star +.. _hashlib-seealso: + .. seealso:: Module :mod:`hmac` @@ -783,15 +826,18 @@ Domain Dedication 1.0 Universal: Module :mod:`base64` Another way to encode binary hashes for non-binary environments. - https://www.blake2.net - Official BLAKE2 website. + https://nvlpubs.nist.gov/nistpubs/fips/nist.fips.180-4.pdf + The FIPS 180-4 publication on Secure Hash Algorithms. - https://csrc.nist.gov/csrc/media/publications/fips/180/2/archive/2002-08-01/documents/fips180-2.pdf - The FIPS 180-2 publication on Secure Hash Algorithms. + https://csrc.nist.gov/publications/detail/fips/202/final + The FIPS 202 publication on the SHA-3 Standard. + + https://www.blake2.net/ + Official BLAKE2 website. - https://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms - Wikipedia article with information on which algorithms have known issues and - what that means regarding their use. + https://en.wikipedia.org/wiki/Cryptographic_hash_function + Wikipedia article with information on which algorithms have known issues + and what that means regarding their use. https://www.ietf.org/rfc/rfc8018.txt PKCS #5: Password-Based Cryptography Specification Version 2.1 diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 03aff25ce6117a..d35090111e0822 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -173,7 +173,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): .. method:: HTMLParser.handle_charref(name) This method is called to process decimal and hexadecimal numeric character - references of the form ``&#NNN;`` and ``&#xNNN;``. For example, the decimal + references of the form :samp:`&#{NNN};` and :samp:`&#x{NNN};`. For example, the decimal equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; in this case the method will receive ``'62'`` or ``'x3E'``. This method is never called if *convert_charrefs* is ``True``. diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index bf1f2e3920783d..c46314fc5e253b 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -83,7 +83,7 @@ The module provides the following classes: .. versionchanged:: 3.2 This class now supports HTTPS virtual hosts if possible (that is, - if :data:`ssl.HAS_SNI` is true). + if :const:`ssl.HAS_SNI` is true). .. versionchanged:: 3.4 The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are @@ -95,16 +95,6 @@ The module provides the following classes: :func:`ssl._create_unverified_context` can be passed to the *context* parameter. - .. deprecated:: 3.6 - *key_file* and *cert_file* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - The *check_hostname* parameter is also deprecated; the - :attr:`ssl.SSLContext.check_hostname` attribute of *context* should - be used instead. - .. versionchanged:: 3.8 This class now enables TLS 1.3 :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or @@ -116,8 +106,8 @@ The module provides the following classes: ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. .. versionchanged:: 3.12 - The deprecated *key_file*, *cert_file* and *check_hostname* parameters - have been removed. + The deprecated *key_file*, *cert_file* and *check_hostname* parameters + have been removed. .. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) @@ -394,6 +384,17 @@ HTTPConnection Objects one will be automatically generated and transmitted if not provided in the headers argument. + +.. method:: HTTPConnection.get_proxy_response_headers() + + Returns a dictionary with the headers of the response received from + the proxy server to the CONNECT request. + + If the CONNECT request was not sent, the method returns ``None``. + + .. versionadded:: 3.12 + + .. method:: HTTPConnection.connect() Connect to the server specified when the object was created. By default, diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 87ef156a0bed57..12a6d768437ea5 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -44,8 +44,8 @@ The module defines the following exception: cookies from a file. :exc:`LoadError` is a subclass of :exc:`OSError`. .. versionchanged:: 3.3 - LoadError was made a subclass of :exc:`OSError` instead of - :exc:`IOError`. + :exc:`LoadError` used to be a subtype of :exc:`IOError`, which is now an + alias of :exc:`OSError`. The following classes are provided: diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index ae75e6dc5fdcf3..6f79b222790094 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -217,7 +217,7 @@ provides three different variants: attribute holds the default values for *message* and *explain* that will be used if no value is provided; for unknown codes the default value for both is the string ``???``. The body will be empty if the method is - HEAD or the response code is one of the following: ``1xx``, + HEAD or the response code is one of the following: :samp:`1{xx}`, ``204 No Content``, ``205 Reset Content``, ``304 Not Modified``. .. versionchanged:: 3.4 @@ -502,11 +502,24 @@ following command runs an HTTP/1.1 conformant server:: Note that CGI scripts will be run with UID of user nobody, for security reasons. Problems with the CGI script will be translated to error 403. + .. deprecated-removed:: 3.13 3.15 + + :class:`CGIHTTPRequestHandler` is being removed in 3.15. CGI has not + been considered a good way to do things for well over a decade. This code + has been unmaintained for a while now and sees very little practical use. + Retaining it could lead to further :ref:`security considerations + `. + :class:`CGIHTTPRequestHandler` can be enabled in the command line by passing the ``--cgi`` option:: python -m http.server --cgi +.. deprecated-removed:: 3.13 3.15 + + :mod:`http.server` command line ``--cgi`` support is being removed + because :class:`CGIHTTPRequestHandler` is being removed. + .. _http.server-security: Security Considerations diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 3058bcead661f3..e710d0bacf3fee 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -439,24 +439,24 @@ the :kbd:`Command` key on macOS. * Some useful Emacs bindings are inherited from Tcl/Tk: - * :kbd:`C-a` beginning of line + * :kbd:`C-a` beginning of line - * :kbd:`C-e` end of line + * :kbd:`C-e` end of line - * :kbd:`C-k` kill line (but doesn't put it in clipboard) + * :kbd:`C-k` kill line (but doesn't put it in clipboard) - * :kbd:`C-l` center window around the insertion point + * :kbd:`C-l` center window around the insertion point - * :kbd:`C-b` go backward one character without deleting (usually you can - also use the cursor key for this) + * :kbd:`C-b` go backward one character without deleting (usually you can + also use the cursor key for this) - * :kbd:`C-f` go forward one character without deleting (usually you can - also use the cursor key for this) + * :kbd:`C-f` go forward one character without deleting (usually you can + also use the cursor key for this) - * :kbd:`C-p` go up one line (usually you can also use the cursor key for - this) + * :kbd:`C-p` go up one line (usually you can also use the cursor key for + this) - * :kbd:`C-d` delete next character + * :kbd:`C-d` delete next character Standard keybindings (like :kbd:`C-c` to copy and :kbd:`C-v` to paste) may work. Keybindings are selected in the Configure IDLE dialog. @@ -479,7 +479,7 @@ Search and Replace Any selection becomes a search target. However, only selections within a line work because searches are only performed within lines with the -terminal newline removed. If ``[x] Regular expresion`` is checked, the +terminal newline removed. If ``[x] Regular expression`` is checked, the target is interpreted according to the Python re module. .. _completions: diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index f1344ae9bb0a49..1f774e64b0eae3 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -106,20 +106,13 @@ There's also a subclass for secure connections: .. versionchanged:: 3.4 The class now supports hostname check with :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *ssl_context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. + :const:`ssl.HAS_SNI`). .. versionchanged:: 3.9 The optional *timeout* parameter was added. .. versionchanged:: 3.12 - The deprecated *keyfile* and *certfile* parameters have been removed. + The deprecated *keyfile* and *certfile* parameters have been removed. The second subclass allows for connections created by a child process: @@ -510,7 +503,7 @@ An :class:`IMAP4` instance has the following methods: .. versionchanged:: 3.4 The method now supports hostname check with :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). + :const:`ssl.HAS_SNI`). .. method:: IMAP4.status(mailbox, names) diff --git a/Doc/library/imghdr.rst b/Doc/library/imghdr.rst deleted file mode 100644 index 630fd7019f94de..00000000000000 --- a/Doc/library/imghdr.rst +++ /dev/null @@ -1,86 +0,0 @@ -:mod:`imghdr` --- Determine the type of an image -================================================ - -.. module:: imghdr - :synopsis: Determine the type of image contained in a file or byte stream. - :deprecated: - -**Source code:** :source:`Lib/imghdr.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`imghdr` module is deprecated - (see :pep:`PEP 594 <594#imghdr>` for details and alternatives). - --------------- - -The :mod:`imghdr` module determines the type of image contained in a file or -byte stream. - -The :mod:`imghdr` module defines the following function: - - -.. function:: what(file, h=None) - - Test the image data contained in the file named *file* and return a - string describing the image type. If *h* is provided, the *file* - argument is ignored and *h* is assumed to contain the byte stream to test. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -The following image types are recognized, as listed below with the return value -from :func:`what`: - -+------------+-----------------------------------+ -| Value | Image format | -+============+===================================+ -| ``'rgb'`` | SGI ImgLib Files | -+------------+-----------------------------------+ -| ``'gif'`` | GIF 87a and 89a Files | -+------------+-----------------------------------+ -| ``'pbm'`` | Portable Bitmap Files | -+------------+-----------------------------------+ -| ``'pgm'`` | Portable Graymap Files | -+------------+-----------------------------------+ -| ``'ppm'`` | Portable Pixmap Files | -+------------+-----------------------------------+ -| ``'tiff'`` | TIFF Files | -+------------+-----------------------------------+ -| ``'rast'`` | Sun Raster Files | -+------------+-----------------------------------+ -| ``'xbm'`` | X Bitmap Files | -+------------+-----------------------------------+ -| ``'jpeg'`` | JPEG data in JFIF or Exif formats | -+------------+-----------------------------------+ -| ``'bmp'`` | BMP files | -+------------+-----------------------------------+ -| ``'png'`` | Portable Network Graphics | -+------------+-----------------------------------+ -| ``'webp'`` | WebP files | -+------------+-----------------------------------+ -| ``'exr'`` | OpenEXR Files | -+------------+-----------------------------------+ - -.. versionadded:: 3.5 - The *exr* and *webp* formats were added. - - -You can extend the list of file types :mod:`imghdr` can recognize by appending -to this variable: - - -.. data:: tests - - A list of functions performing the individual tests. Each function takes two - arguments: the byte-stream and an open file-like object. When :func:`what` is - called with a byte-stream, the file-like object will be ``None``. - - The test function should return a string describing the image type if the test - succeeded, or ``None`` if it failed. - -Example:: - - >>> import imghdr - >>> imghdr.what('bass.gif') - 'gif' - diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index d2cc769e2c8400..1df7d8d772a274 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -41,7 +41,7 @@ and metadata defined by the `Core metadata specifications ` + You can use :ref:`packages_distributions() ` to get a mapping between them. By default, distribution metadata can live on the file system diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 2d0f137ffc7996..c508b6ba965cc0 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -43,8 +43,6 @@ :const:`None`. An object compatible with this ABC should only be returned when the specified module is a package. - .. versionadded:: 3.7 - .. deprecated-removed:: 3.12 3.14 Use :class:`importlib.resources.abc.TraversableResources` instead. @@ -95,11 +93,6 @@ For a representation of the object on the file-system, use :meth:`importlib.resources.as_file`. - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.Traversable` instead. - .. attribute:: name Abstract. The base name of this object without any parent references. @@ -130,7 +123,7 @@ suitable for reading (same as :attr:`pathlib.Path.open`). When opening as text, accepts encoding parameters such as those - accepted by :attr:`io.TextIOWrapper`. + accepted by :class:`io.TextIOWrapper`. .. method:: read_bytes() @@ -145,19 +138,14 @@ An abstract base class for resource readers capable of serving the :meth:`importlib.resources.files` interface. Subclasses - :class:`importlib.resources.abc.ResourceReader` and provides - concrete implementations of the :class:`importlib.resources.abc.ResourceReader`'s + :class:`ResourceReader` and provides + concrete implementations of the :class:`!ResourceReader`'s abstract methods. Therefore, any loader supplying - :class:`importlib.abc.TraversableResources` also supplies ResourceReader. + :class:`!TraversableResources` also supplies :class:`!ResourceReader`. Loaders that wish to support resource reading are expected to implement this interface. - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - .. abstractmethod:: files() Returns a :class:`importlib.resources.abc.Traversable` object for the loaded diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 755693840fecd8..a5adf0b8546dbf 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -50,7 +50,7 @@ for example, a package and its resources can be imported from a zip file using ``get_resource_reader(fullname)`` method as specified by :class:`importlib.resources.abc.ResourceReader`. -.. data:: Anchor +.. class:: Anchor Represents an anchor for resources, either a :class:`module object ` or a module name as a string. Defined as @@ -63,7 +63,7 @@ for example, a package and its resources can be imported from a zip file using (think files). A Traversable may contain other containers (think subdirectories). - *anchor* is an optional :data:`Anchor`. If the anchor is a + *anchor* is an optional :class:`Anchor`. If the anchor is a package, resources are resolved from that package. If a module, resources are resolved adjacent to that module (in the same package or the package root). If the anchor is omitted, the caller's module @@ -72,181 +72,28 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 .. versionchanged:: 3.12 - "package" parameter was renamed to "anchor". "anchor" can now + *package* parameter was renamed to *anchor*. *anchor* can now be a non-package module and if omitted will default to the caller's - module. "package" is still accepted for compatibility but will raise - a DeprecationWarning. Consider passing the anchor positionally or + module. *package* is still accepted for compatibility but will raise + a :exc:`DeprecationWarning`. Consider passing the anchor positionally or using ``importlib_resources >= 5.10`` for a compatible interface on older Pythons. .. function:: as_file(traversable) Given a :class:`~importlib.resources.abc.Traversable` object representing - a file, typically from :func:`importlib.resources.files`, return - a context manager for use in a :keyword:`with` statement. + a file or directory, typically from :func:`importlib.resources.files`, + return a context manager for use in a :keyword:`with` statement. The context manager provides a :class:`pathlib.Path` object. - Exiting the context manager cleans up any temporary file created when the - resource was extracted from e.g. a zip file. + Exiting the context manager cleans up any temporary file or directory + created when the resource was extracted from e.g. a zip file. Use ``as_file`` when the Traversable methods - (``read_text``, etc) are insufficient and an actual file on + (``read_text``, etc) are insufficient and an actual file or directory on the file system is required. .. versionadded:: 3.9 - -Deprecated functions -^^^^^^^^^^^^^^^^^^^^ - -An older, deprecated set of functions is still available, but is -scheduled for removal in a future version of Python. -The main drawback of these functions is that they do not support -directories: they assume all resources are located directly within a *package*. - -.. data:: Package - - Whenever a function accepts a ``Package`` argument, you can pass in - either a :class:`module object ` or a module name - as a string. You can only pass module objects whose - ``__spec__.submodule_search_locations`` is not ``None``. - - The ``Package`` type is defined as ``Union[str, ModuleType]``. - - .. deprecated:: 3.12 - - -.. data:: Resource - - For *resource* arguments of the functions below, you can pass in - the name of a resource as a string or - a :class:`path-like object `. - - The ``Resource`` type is defined as ``Union[str, os.PathLike]``. - - -.. function:: open_binary(package, resource) - - Open for binary reading the *resource* within *package*. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). This function returns a - ``typing.BinaryIO`` instance, a binary I/O stream open for reading. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).open('rb') - - -.. function:: open_text(package, resource, encoding='utf-8', errors='strict') - - Open for text reading the *resource* within *package*. By default, the - resource is opened for reading as UTF-8. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). *encoding* and *errors* - have the same meaning as with built-in :func:`open`. - - This function returns a ``typing.TextIO`` instance, a text I/O stream open - for reading. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).open('r', encoding=encoding) - - -.. function:: read_binary(package, resource) - - Read and return the contents of the *resource* within *package* as - ``bytes``. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). This function returns the - contents of the resource as :class:`bytes`. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).read_bytes() - - -.. function:: read_text(package, resource, encoding='utf-8', errors='strict') - - Read and return the contents of *resource* within *package* as a ``str``. - By default, the contents are read as strict UTF-8. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). *encoding* and *errors* - have the same meaning as with built-in :func:`open`. This function - returns the contents of the resource as :class:`str`. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).read_text(encoding=encoding) - - -.. function:: path(package, resource) - - Return the path to the *resource* as an actual file system path. This - function returns a context manager for use in a :keyword:`with` statement. - The context manager provides a :class:`pathlib.Path` object. - - Exiting the context manager cleans up any temporary file created when the - resource needs to be extracted from e.g. a zip file. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). - - .. deprecated:: 3.11 - - Calls to this function can be replaced using :func:`as_file`:: - - as_file(files(package).joinpath(resource)) - - -.. function:: is_resource(package, name) - - Return ``True`` if there is a resource named *name* in the package, - otherwise ``False``. - This function does not consider directories to be resources. - *package* is either a name or a module object which conforms to the - ``Package`` requirements. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).is_file() - - -.. function:: contents(package) - - Return an iterable over the named items within the package. The iterable - returns :class:`str` resources (e.g. files) and non-resources - (e.g. directories). The iterable does not recurse into subdirectories. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - (resource.name for resource in files(package).iterdir() if resource.is_file()) + .. versionchanged:: 3.12 + Added support for *traversable* representing a directory. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 65aaad0df9ee66..2402bc5cd3ee2c 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -372,7 +372,7 @@ ABC hierarchy:: The list of locations where the package's submodules will be found. Most of the time this is a single directory. The import system passes this attribute to ``__import__()`` and to finders - in the same way as :attr:`sys.path` but just for the package. + in the same way as :data:`sys.path` but just for the package. It is not set on non-package modules so it can be used as an indicator that the module is a package. @@ -609,7 +609,7 @@ ABC hierarchy:: automatically. When writing to the path fails because the path is read-only - (:attr:`errno.EACCES`/:exc:`PermissionError`), do not propagate the + (:const:`errno.EACCES`/:exc:`PermissionError`), do not propagate the exception. .. versionchanged:: 3.4 @@ -645,6 +645,160 @@ ABC hierarchy:: itself does not end in ``__init__``. +.. class:: ResourceReader + + *Superseded by TraversableResources* + + An :term:`abstract base class` to provide the ability to read + *resources*. + + From the perspective of this ABC, a *resource* is a binary + artifact that is shipped within a package. Typically this is + something like a data file that lives next to the ``__init__.py`` + file of the package. The purpose of this class is to help abstract + out the accessing of such data files so that it does not matter if + the package and its data file(s) are stored in a e.g. zip file + versus on the file system. + + For any of methods of this class, a *resource* argument is + expected to be a :term:`path-like object` which represents + conceptually just a file name. This means that no subdirectory + paths should be included in the *resource* argument. This is + because the location of the package the reader is for, acts as the + "directory". Hence the metaphor for directories and file + names is packages and resources, respectively. This is also why + instances of this class are expected to directly correlate to + a specific package (instead of potentially representing multiple + packages or a module). + + Loaders that wish to support resource reading are expected to + provide a method called ``get_resource_reader(fullname)`` which + returns an object implementing this ABC's interface. If the module + specified by fullname is not a package, this method should return + :const:`None`. An object compatible with this ABC should only be + returned when the specified module is a package. + + .. versionadded:: 3.7 + + .. deprecated-removed:: 3.12 3.14 + Use :class:`importlib.resources.abc.TraversableResources` instead. + + .. abstractmethod:: open_resource(resource) + + Returns an opened, :term:`file-like object` for binary reading + of the *resource*. + + If the resource cannot be found, :exc:`FileNotFoundError` is + raised. + + .. abstractmethod:: resource_path(resource) + + Returns the file system path to the *resource*. + + If the resource does not concretely exist on the file system, + raise :exc:`FileNotFoundError`. + + .. abstractmethod:: is_resource(name) + + Returns ``True`` if the named *name* is considered a resource. + :exc:`FileNotFoundError` is raised if *name* does not exist. + + .. abstractmethod:: contents() + + Returns an :term:`iterable` of strings over the contents of + the package. Do note that it is not required that all names + returned by the iterator be actual resources, e.g. it is + acceptable to return names for which :meth:`is_resource` would + be false. + + Allowing non-resource names to be returned is to allow for + situations where how a package and its resources are stored + are known a priori and the non-resource names would be useful. + For instance, returning subdirectory names is allowed so that + when it is known that the package and resources are stored on + the file system then those subdirectory names can be used + directly. + + The abstract method returns an iterable of no items. + + +.. class:: Traversable + + An object with a subset of :class:`pathlib.Path` methods suitable for + traversing directories and opening files. + + For a representation of the object on the file-system, use + :meth:`importlib.resources.as_file`. + + .. versionadded:: 3.9 + + .. deprecated-removed:: 3.12 3.14 + Use :class:`importlib.resources.abc.Traversable` instead. + + .. attribute:: name + + Abstract. The base name of this object without any parent references. + + .. abstractmethod:: iterdir() + + Yield ``Traversable`` objects in ``self``. + + .. abstractmethod:: is_dir() + + Return ``True`` if ``self`` is a directory. + + .. abstractmethod:: is_file() + + Return ``True`` if ``self`` is a file. + + .. abstractmethod:: joinpath(child) + + Return Traversable child in ``self``. + + .. abstractmethod:: __truediv__(child) + + Return ``Traversable`` child in ``self``. + + .. abstractmethod:: open(mode='r', *args, **kwargs) + + *mode* may be 'r' or 'rb' to open as text or binary. Return a handle + suitable for reading (same as :attr:`pathlib.Path.open`). + + When opening as text, accepts encoding parameters such as those + accepted by :attr:`io.TextIOWrapper`. + + .. method:: read_bytes() + + Read contents of ``self`` as bytes. + + .. method:: read_text(encoding=None) + + Read contents of ``self`` as text. + + +.. class:: TraversableResources + + An abstract base class for resource readers capable of serving + the :meth:`importlib.resources.files` interface. Subclasses + :class:`importlib.resources.abc.ResourceReader` and provides + concrete implementations of the :class:`importlib.resources.abc.ResourceReader`'s + abstract methods. Therefore, any loader supplying + :class:`importlib.abc.TraversableResources` also supplies ResourceReader. + + Loaders that wish to support resource reading are expected to + implement this interface. + + .. versionadded:: 3.9 + + .. deprecated-removed:: 3.12 3.14 + Use :class:`importlib.resources.abc.TraversableResources` instead. + + .. abstractmethod:: files() + + Returns a :class:`importlib.resources.abc.Traversable` object for the loaded + package. + + :mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------ @@ -843,7 +997,7 @@ find and load modules. .. classmethod:: path_hook(*loader_details) - A class method which returns a closure for use on :attr:`sys.path_hooks`. + A class method which returns a closure for use on :data:`sys.path_hooks`. An instance of :class:`FileFinder` is returned by the closure using the path argument given to the closure directly and *loader_details* indirectly. @@ -941,8 +1095,15 @@ find and load modules. The *fullname* argument specifies the name of the module the loader is to support. The *path* argument is the path to the extension module's file. + Note that, by default, importing an extension module will fail + in subinterpreters if it doesn't implement multi-phase init + (see :pep:`489`), even if it would otherwise import successfully. + .. versionadded:: 3.3 + .. versionchanged:: 3.12 + Multi-phase init is now required for use in subinterpreters. + .. attribute:: name Name of the module the loader supports. @@ -984,7 +1145,7 @@ find and load modules. .. versionadded:: 3.4 -.. class:: NamespaceLoader(name, path, path_finder): +.. class:: NamespaceLoader(name, path, path_finder) A concrete implementation of :class:`importlib.abc.InspectLoader` for namespace packages. This is an alias for a private class and is only made @@ -1184,10 +1345,10 @@ an :term:`importer`. .. function:: find_spec(name, package=None) Find the :term:`spec ` for a module, optionally relative to - the specified **package** name. If the module is in :attr:`sys.modules`, + the specified **package** name. If the module is in :data:`sys.modules`, then ``sys.modules[name].__spec__`` is returned (unless the spec would be ``None`` or is not set, in which case :exc:`ValueError` is raised). - Otherwise a search using :attr:`sys.meta_path` is done. ``None`` is + Otherwise a search using :data:`sys.meta_path` is done. ``None`` is returned if no spec is found. If **name** is for a submodule (contains a dot), the parent module is @@ -1248,6 +1409,30 @@ an :term:`importer`. .. versionadded:: 3.7 +.. function:: _incompatible_extension_module_restrictions(*, disable_check) + + A context manager that can temporarily skip the compatibility check + for extension modules. By default the check is enabled and will fail + when a single-phase init module is imported in a subinterpreter. + It will also fail for a multi-phase init module that doesn't + explicitly support a per-interpreter GIL, when imported + in an interpreter with its own GIL. + + Note that this function is meant to accommodate an unusual case; + one which is likely to eventually go away. There's is a pretty good + chance this is not what you were looking for. + + You can get the same effect as this function by implementing the + basic interface of multi-phase init (:pep:`489`) and lying about + support for multiple interpreters (or per-interpreter GIL). + + .. warning:: + Using this function to disable the check can lead to + unexpected behavior and even crashes. It should only be used during + extension module development. + + .. versionadded:: 3.12 + .. class:: LazyLoader(loader) A class which postpones the execution of the loader of a module until the @@ -1259,7 +1444,7 @@ an :term:`importer`. :meth:`~importlib.abc.Loader.create_module` method must return ``None`` or a type for which its ``__class__`` attribute can be mutated along with not using :term:`slots <__slots__>`. Finally, modules which substitute the object - placed into :attr:`sys.modules` will not work as there is no way to properly + placed into :data:`sys.modules` will not work as there is no way to properly replace the module references throughout the interpreter safely; :exc:`ValueError` is raised if such a substitution is detected. @@ -1383,9 +1568,9 @@ For deep customizations of import, you typically want to implement an :term:`importer`. This means managing both the :term:`finder` and :term:`loader` side of things. For finders there are two flavours to choose from depending on your needs: a :term:`meta path finder` or a :term:`path entry finder`. The -former is what you would put on :attr:`sys.meta_path` while the latter is what -you create using a :term:`path entry hook` on :attr:`sys.path_hooks` which works -with :attr:`sys.path` entries to potentially create a finder. This example will +former is what you would put on :data:`sys.meta_path` while the latter is what +you create using a :term:`path entry hook` on :data:`sys.path_hooks` which works +with :data:`sys.path` entries to potentially create a finder. This example will show you how to register your own importers so that import will use them (for creating an importer for yourself, read the documentation for the appropriate classes defined within this package):: diff --git a/Doc/library/index.rst b/Doc/library/index.rst index d064b680f9aaa4..0b348ae6f5c8c0 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -73,5 +73,6 @@ the `Python Package Index `_. language.rst windows.rst unix.rst + cmdline.rst superseded.rst security_warnings.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 7884308a333020..8381e508139fbd 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1,6 +1,11 @@ :mod:`inspect` --- Inspect live objects ======================================= +.. testsetup:: * + + import inspect + from inspect import * + .. module:: inspect :synopsis: Extract information and source code from live objects. @@ -268,7 +273,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): :func:`getmembers` will only return class attributes defined in the metaclass when the argument is a class and those attributes have been - listed in the metaclass' custom :meth:`__dir__`. + listed in the metaclass' custom :meth:`~object.__dir__`. .. function:: getmembers_static(object[, predicate]) @@ -387,7 +392,11 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object can be used in :keyword:`await` expression. Can also be used to distinguish generator-based coroutines from regular - generators:: + generators: + + .. testcode:: + + import types def gen(): yield @@ -404,13 +413,15 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: isasyncgenfunction(object) Return ``True`` if the object is an :term:`asynchronous generator` function, - for example:: + for example: - >>> async def agen(): - ... yield 1 - ... - >>> inspect.isasyncgenfunction(agen) - True + .. doctest:: + + >>> async def agen(): + ... yield 1 + ... + >>> inspect.isasyncgenfunction(agen) + True .. versionadded:: 3.6 @@ -476,12 +487,13 @@ attributes (see :ref:`import-mod-attrs` for module attributes): has a :meth:`~object.__get__` method but not a :meth:`~object.__set__` method, but beyond that the set of attributes varies. A :attr:`~definition.__name__` attribute is usually - sensible, and :attr:`__doc__` often is. + sensible, and :attr:`!__doc__` often is. Methods implemented via descriptors that also pass one of the other tests return ``False`` from the :func:`ismethoddescriptor` test, simply because the other tests promise more -- you can, e.g., count on having the - :attr:`__func__` attribute (etc) when an object passes :func:`ismethod`. + :attr:`~method.__func__` attribute (etc) when an object passes + :func:`ismethod`. .. function:: isdatadescriptor(object) @@ -492,7 +504,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Examples are properties (defined in Python), getsets, and members. The latter two are defined in C and there are more specific tests available for those types, which is robust across Python implementations. Typically, data - descriptors will also have :attr:`~definition.__name__` and :attr:`__doc__` attributes + descriptors will also have :attr:`~definition.__name__` and :attr:`!__doc__` attributes (properties, getsets, and members have both of these attributes), but this is not guaranteed. @@ -614,13 +626,16 @@ Introspecting callables with the Signature object .. versionadded:: 3.3 -The Signature object represents the call signature of a callable object and its -return annotation. To retrieve a Signature object, use the :func:`signature` +The :class:`Signature` object represents the call signature of a callable object +and its return annotation. To retrieve a :class:`!Signature` object, +use the :func:`!signature` function. .. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) - Return a :class:`Signature` object for the given ``callable``:: + Return a :class:`Signature` object for the given *callable*: + + .. doctest:: >>> from inspect import signature >>> def foo(a, *, b:int, **kwargs): @@ -629,10 +644,10 @@ function. >>> sig = signature(foo) >>> str(sig) - '(a, *, b:int, **kwargs)' + '(a, *, b: int, **kwargs)' >>> str(sig.parameters['b']) - 'b:int' + 'b: int' >>> sig.parameters['b'].annotation @@ -640,32 +655,36 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. + If the passed object has a ``__signature__`` attribute, this function + returns it without further computations. + For objects defined in modules using stringized annotations (``from __future__ import annotations``), :func:`signature` will attempt to automatically un-stringize the annotations using - :func:`inspect.get_annotations()`. The - ``global``, ``locals``, and ``eval_str`` parameters are passed - into :func:`inspect.get_annotations()` when resolving the - annotations; see the documentation for :func:`inspect.get_annotations()` + :func:`get_annotations`. The + *globals*, *locals*, and *eval_str* parameters are passed + into :func:`get_annotations` when resolving the + annotations; see the documentation for :func:`get_annotations` for instructions on how to use these parameters. Raises :exc:`ValueError` if no signature can be provided, and :exc:`TypeError` if that type of object is not supported. Also, - if the annotations are stringized, and ``eval_str`` is not false, - the ``eval()`` call(s) to un-stringize the annotations could - potentially raise any kind of exception. + if the annotations are stringized, and *eval_str* is not false, + the ``eval()`` call(s) to un-stringize the annotations in :func:`get_annotations` + could potentially raise any kind of exception. A slash(/) in the signature of a function denotes that the parameters prior to it are positional-only. For more info, see :ref:`the FAQ entry on positional-only parameters `. - .. versionadded:: 3.5 - ``follow_wrapped`` parameter. Pass ``False`` to get a signature of - ``callable`` specifically (``callable.__wrapped__`` will not be used to + .. versionchanged:: 3.5 + The *follow_wrapped* parameter was added. + Pass ``False`` to get a signature of + *callable* specifically (``callable.__wrapped__`` will not be used to unwrap decorated callables.) - .. versionadded:: 3.10 - ``globals``, ``locals``, and ``eval_str`` parameters. + .. versionchanged:: 3.10 + The *globals*, *locals*, and *eval_str* parameters were added. .. note:: @@ -676,7 +695,8 @@ function. .. class:: Signature(parameters=None, *, return_annotation=Signature.empty) - A Signature object represents the call signature of a function and its return + A :class:`!Signature` object represents the call signature of a function + and its return annotation. For each parameter accepted by the function it stores a :class:`Parameter` object in its :attr:`parameters` collection. @@ -686,14 +706,14 @@ function. positional-only first, then positional-or-keyword, and that parameters with defaults follow parameters without defaults. - The optional *return_annotation* argument, can be an arbitrary Python object, - is the "return" annotation of the callable. + The optional *return_annotation* argument can be an arbitrary Python object. + It represents the "return" annotation of the callable. - Signature objects are *immutable*. Use :meth:`Signature.replace` to make a - modified copy. + :class:`!Signature` objects are *immutable*. Use :meth:`Signature.replace` or + :func:`copy.replace` to make a modified copy. .. versionchanged:: 3.5 - Signature objects are picklable and :term:`hashable`. + :class:`!Signature` objects are now picklable and :term:`hashable`. .. attribute:: Signature.empty @@ -730,13 +750,15 @@ function. .. method:: Signature.replace(*[, parameters][, return_annotation]) - Create a new Signature instance based on the instance replace was invoked - on. It is possible to pass different ``parameters`` and/or - ``return_annotation`` to override the corresponding properties of the base - signature. To remove return_annotation from the copied Signature, pass in + Create a new :class:`Signature` instance based on the instance + :meth:`replace` was invoked on. + It is possible to pass different *parameters* and/or + *return_annotation* to override the corresponding properties of the base + signature. To remove ``return_annotation`` from the copied + :class:`!Signature`, pass in :attr:`Signature.empty`. - :: + .. doctest:: >>> def test(a, b): ... pass @@ -746,33 +768,50 @@ function. >>> str(new_sig) "(a, b) -> 'new return anno'" - .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globalns=None, localns=None) + :class:`Signature` objects are also supported by the generic function + :func:`copy.replace`. + + .. method:: format(*, max_width=None) + + Create a string representation of the :class:`Signature` object. + + If *max_width* is passed, the method will attempt to fit + the signature into lines of at most *max_width* characters. + If the signature is longer than *max_width*, + all parameters will be on separate lines. + + .. versionadded:: 3.13 + + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) Return a :class:`Signature` (or its subclass) object for a given callable - ``obj``. Pass ``follow_wrapped=False`` to get a signature of ``obj`` - without unwrapping its ``__wrapped__`` chain. ``globalns`` and - ``localns`` will be used as the namespaces when resolving annotations. + *obj*. + + This method simplifies subclassing of :class:`Signature`: - This method simplifies subclassing of :class:`Signature`:: + .. testcode:: - class MySignature(Signature): - pass - sig = MySignature.from_callable(min) - assert isinstance(sig, MySignature) + class MySignature(Signature): + pass + sig = MySignature.from_callable(sum) + assert isinstance(sig, MySignature) + + Its behavior is otherwise identical to that of :func:`signature`. .. versionadded:: 3.5 - .. versionadded:: 3.10 - ``globalns`` and ``localns`` parameters. + .. versionchanged:: 3.10 + The *globals*, *locals*, and *eval_str* parameters were added. .. class:: Parameter(name, kind, *, default=Parameter.empty, annotation=Parameter.empty) - Parameter objects are *immutable*. Instead of modifying a Parameter object, - you can use :meth:`Parameter.replace` to create a modified copy. + :class:`!Parameter` objects are *immutable*. + Instead of modifying a :class:`!Parameter` object, + you can use :meth:`Parameter.replace` or :func:`copy.replace` to create a modified copy. .. versionchanged:: 3.5 - Parameter objects are picklable and :term:`hashable`. + Parameter objects are now picklable and :term:`hashable`. .. attribute:: Parameter.empty @@ -791,7 +830,7 @@ function. expressions. .. versionchanged:: 3.6 - These parameter names are exposed by this module as names like + These parameter names are now exposed by this module as names like ``implicit0``. .. attribute:: Parameter.default @@ -841,7 +880,9 @@ function. | | definition. | +------------------------+----------------------------------------------+ - Example: print all keyword-only arguments without default values:: + Example: print all keyword-only arguments without default values: + + .. doctest:: >>> def foo(a, b, *, c, d=10): ... pass @@ -855,11 +896,13 @@ function. .. attribute:: Parameter.kind.description - Describes a enum value of Parameter.kind. + Describes a enum value of :attr:`Parameter.kind`. .. versionadded:: 3.8 - Example: print all descriptions of arguments:: + Example: print all descriptions of arguments: + + .. doctest:: >>> def foo(a, b, *, c, d=10): ... pass @@ -874,12 +917,12 @@ function. .. method:: Parameter.replace(*[, name][, kind][, default][, annotation]) - Create a new Parameter instance based on the instance replaced was invoked - on. To override a :class:`Parameter` attribute, pass the corresponding + Create a new :class:`Parameter` instance based on the instance replaced was invoked + on. To override a :class:`!Parameter` attribute, pass the corresponding argument. To remove a default value or/and an annotation from a - Parameter, pass :attr:`Parameter.empty`. + :class:`!Parameter`, pass :attr:`Parameter.empty`. - :: + .. doctest:: >>> from inspect import Parameter >>> param = Parameter('foo', Parameter.KEYWORD_ONLY, default=42) @@ -890,10 +933,13 @@ function. 'foo=42' >>> str(param.replace(default=Parameter.empty, annotation='spam')) - "foo:'spam'" + "foo: 'spam'" + + :class:`Parameter` objects are also supported by the generic function + :func:`copy.replace`. .. versionchanged:: 3.4 - In Python 3.3 Parameter objects were allowed to have ``name`` set + In Python 3.3 :class:`Parameter` objects were allowed to have ``name`` set to ``None`` if their ``kind`` was set to ``POSITIONAL_ONLY``. This is no longer permitted. @@ -946,18 +992,20 @@ function. For variable-keyword arguments (``**kwargs``) the default is an empty dict. - :: + .. doctest:: - >>> def foo(a, b='ham', *args): pass - >>> ba = inspect.signature(foo).bind('spam') - >>> ba.apply_defaults() - >>> ba.arguments - {'a': 'spam', 'b': 'ham', 'args': ()} + >>> def foo(a, b='ham', *args): pass + >>> ba = inspect.signature(foo).bind('spam') + >>> ba.apply_defaults() + >>> ba.arguments + {'a': 'spam', 'b': 'ham', 'args': ()} .. versionadded:: 3.5 The :attr:`args` and :attr:`kwargs` properties can be used to invoke - functions:: + functions: + + .. testcode:: def test(a, *, b): ... @@ -1076,20 +1124,22 @@ Classes and functions ``**`` arguments, if any) to their values from *args* and *kwds*. In case of invoking *func* incorrectly, i.e. whenever ``func(*args, **kwds)`` would raise an exception because of incompatible signature, an exception of the same type - and the same or similar message is raised. For example:: - - >>> from inspect import getcallargs - >>> def f(a, b=1, *pos, **named): - ... pass - ... - >>> getcallargs(f, 1, 2, 3) == {'a': 1, 'named': {}, 'b': 2, 'pos': (3,)} - True - >>> getcallargs(f, a=2, x=4) == {'a': 2, 'named': {'x': 4}, 'b': 1, 'pos': ()} - True - >>> getcallargs(f) - Traceback (most recent call last): - ... - TypeError: f() missing 1 required positional argument: 'a' + and the same or similar message is raised. For example: + + .. doctest:: + + >>> from inspect import getcallargs + >>> def f(a, b=1, *pos, **named): + ... pass + ... + >>> getcallargs(f, 1, 2, 3) == {'a': 1, 'named': {}, 'b': 2, 'pos': (3,)} + True + >>> getcallargs(f, a=2, x=4) == {'a': 2, 'named': {'x': 4}, 'b': 1, 'pos': ()} + True + >>> getcallargs(f) + Traceback (most recent call last): + ... + TypeError: f() missing 1 required positional argument: 'a' .. versionadded:: 3.2 @@ -1391,7 +1441,8 @@ Fetching attributes statically Both :func:`getattr` and :func:`hasattr` can trigger code execution when fetching or checking for the existence of attributes. Descriptors, like -properties, will be invoked and :meth:`__getattr__` and :meth:`__getattribute__` +properties, will be invoked and :meth:`~object.__getattr__` and +:meth:`~object.__getattribute__` may be called. For cases where you want passive introspection, like documentation tools, this @@ -1401,7 +1452,8 @@ but avoids executing code when it fetches attributes. .. function:: getattr_static(obj, attr, default=None) Retrieve attributes without triggering dynamic lookup via the - descriptor protocol, :meth:`__getattr__` or :meth:`__getattribute__`. + descriptor protocol, :meth:`~object.__getattr__` + or :meth:`~object.__getattribute__`. Note: this function may not be able to retrieve all attributes that getattr can fetch (like dynamically created attributes) @@ -1458,10 +1510,11 @@ generator to be determined easily. Get current state of a generator-iterator. Possible states are: - * GEN_CREATED: Waiting to start execution. - * GEN_RUNNING: Currently being executed by the interpreter. - * GEN_SUSPENDED: Currently suspended at a yield expression. - * GEN_CLOSED: Execution has completed. + + * GEN_CREATED: Waiting to start execution. + * GEN_RUNNING: Currently being executed by the interpreter. + * GEN_SUSPENDED: Currently suspended at a yield expression. + * GEN_CLOSED: Execution has completed. .. versionadded:: 3.2 @@ -1473,10 +1526,11 @@ generator to be determined easily. ``cr_frame`` attributes. Possible states are: - * CORO_CREATED: Waiting to start execution. - * CORO_RUNNING: Currently being executed by the interpreter. - * CORO_SUSPENDED: Currently suspended at an await expression. - * CORO_CLOSED: Execution has completed. + + * CORO_CREATED: Waiting to start execution. + * CORO_RUNNING: Currently being executed by the interpreter. + * CORO_SUSPENDED: Currently suspended at an await expression. + * CORO_CLOSED: Execution has completed. .. versionadded:: 3.5 @@ -1489,10 +1543,11 @@ generator to be determined easily. ``ag_running`` and ``ag_frame`` attributes. Possible states are: - * AGEN_CREATED: Waiting to start execution. - * AGEN_RUNNING: Currently being executed by the interpreter. - * AGEN_SUSPENDED: Currently suspended at a yield expression. - * AGEN_CLOSED: Execution has completed. + + * AGEN_CREATED: Waiting to start execution. + * AGEN_RUNNING: Currently being executed by the interpreter. + * AGEN_SUSPENDED: Currently suspended at a yield expression. + * AGEN_CLOSED: Execution has completed. .. versionadded:: 3.12 @@ -1541,8 +1596,8 @@ updated as expected: Code Objects Bit Flags ---------------------- -Python code objects have a ``co_flags`` attribute, which is a bitmap of -the following flags: +Python code objects have a :attr:`~codeobject.co_flags` attribute, +which is a bitmap of the following flags: .. data:: CO_OPTIMIZED @@ -1550,8 +1605,8 @@ the following flags: .. data:: CO_NEWLOCALS - If set, a new dict will be created for the frame's ``f_locals`` when - the code object is executed. + If set, a new dict will be created for the frame's :attr:`~frame.f_locals` + when the code object is executed. .. data:: CO_VARARGS @@ -1650,6 +1705,6 @@ By default, accepts the name of a module and prints the source of that module. A class or function within the module can be printed instead by appended a colon and the qualified name of the target object. -.. cmdoption:: --details +.. option:: --details Print information about the specified object rather than the source code diff --git a/Doc/library/io.rst b/Doc/library/io.rst index c9249da1c3c3d2..6736aa9ee2b0ef 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -38,9 +38,9 @@ location), or only sequential access (for example in the case of a socket or pipe). All streams are careful about the type of data you give to them. For example -giving a :class:`str` object to the ``write()`` method of a binary stream +giving a :class:`str` object to the :meth:`!write` method of a binary stream will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the -``write()`` method of a text stream. +:meth:`!write` method of a text stream. .. versionchanged:: 3.3 Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since @@ -146,7 +146,7 @@ Opt-in EncodingWarning See :pep:`597` for more details. To find where the default locale encoding is used, you can enable -the ``-X warn_default_encoding`` command line option or set the +the :option:`-X warn_default_encoding <-X>` command line option or set the :envvar:`PYTHONWARNDEFAULTENCODING` environment variable, which will emit an :exc:`EncodingWarning` when the default encoding is used. @@ -175,7 +175,7 @@ High-level Module Interface .. audit-event:: open path,mode,flags io.open This function raises an :ref:`auditing event ` ``open`` with - arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` + arguments *path*, *mode* and *flags*. The *mode* and *flags* arguments may have been modified or inferred from the original call. @@ -184,10 +184,10 @@ High-level Module Interface Opens the provided file with mode ``'rb'``. This function should be used when the intent is to treat the contents as executable code. - ``path`` should be a :class:`str` and an absolute path. + *path* should be a :class:`str` and an absolute path. The behavior of this function may be overridden by an earlier call to the - :c:func:`PyFile_SetOpenCodeHook`. However, assuming that ``path`` is a + :c:func:`PyFile_SetOpenCodeHook`. However, assuming that *path* is a :class:`str` and an absolute path, ``open_code(path)`` should always behave the same as ``open(path, 'rb')``. Overriding the behavior is intended for additional validation or preprocessing of the file. @@ -253,12 +253,12 @@ The implementation of I/O streams is organized as a hierarchy of classes. First specify the various categories of streams, then concrete classes providing the standard stream implementations. - .. note:: +.. note:: - The abstract base classes also provide default implementations of some - methods in order to help implementation of concrete stream classes. For - example, :class:`BufferedIOBase` provides unoptimized implementations of - :meth:`~IOBase.readinto` and :meth:`~IOBase.readline`. + The abstract base classes also provide default implementations of some + methods in order to help implementation of concrete stream classes. For + example, :class:`BufferedIOBase` provides unoptimized implementations of + :meth:`!readinto` and :meth:`!readline`. At the top of the I/O hierarchy is the abstract base class :class:`IOBase`. It defines the basic interface to a stream. Note, however, that there is no @@ -320,8 +320,8 @@ I/O Base Classes implementations represent a file that cannot be read, written or seeked. - Even though :class:`IOBase` does not declare :meth:`read` - or :meth:`write` because their signatures will vary, implementations and + Even though :class:`IOBase` does not declare :meth:`!read` + or :meth:`!write` because their signatures will vary, implementations and clients should consider those methods part of the interface. Also, implementations may raise a :exc:`ValueError` (or :exc:`UnsupportedOperation`) when operations they do not support are called. @@ -379,8 +379,8 @@ I/O Base Classes .. method:: readable() - Return ``True`` if the stream can be read from. If ``False``, :meth:`read` - will raise :exc:`OSError`. + Return ``True`` if the stream can be read from. + If ``False``, :meth:`!read` will raise :exc:`OSError`. .. method:: readline(size=-1, /) @@ -401,29 +401,28 @@ I/O Base Classes hint. Note that it's already possible to iterate on file objects using ``for - line in file: ...`` without calling ``file.readlines()``. + line in file: ...`` without calling :meth:`!file.readlines`. - .. method:: seek(offset, whence=SEEK_SET, /) + .. method:: seek(offset, whence=os.SEEK_SET, /) - Change the stream position to the given byte *offset*. *offset* is - interpreted relative to the position indicated by *whence*. The default - value for *whence* is :data:`SEEK_SET`. Values for *whence* are: + Change the stream position to the given byte *offset*, + interpreted relative to the position indicated by *whence*, + and return the new absolute position. + Values for *whence* are: - * :data:`SEEK_SET` or ``0`` -- start of the stream (the default); + * :data:`os.SEEK_SET` or ``0`` -- start of the stream (the default); *offset* should be zero or positive - * :data:`SEEK_CUR` or ``1`` -- current stream position; *offset* may - be negative - * :data:`SEEK_END` or ``2`` -- end of the stream; *offset* is usually - negative - - Return the new absolute position. + * :data:`os.SEEK_CUR` or ``1`` -- current stream position; + *offset* may be negative + * :data:`os.SEEK_END` or ``2`` -- end of the stream; + *offset* is usually negative .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. versionadded:: 3.3 Some operating systems could support additional values, like - :data:`os.SEEK_HOLE` or :data:`os.SEEK_DATA`. The valid values + :const:`os.SEEK_HOLE` or :const:`os.SEEK_DATA`. The valid values for a file could depend on it being open in text or binary mode. .. method:: seekable() @@ -450,7 +449,7 @@ I/O Base Classes .. method:: writable() Return ``True`` if the stream supports writing. If ``False``, - :meth:`write` and :meth:`truncate` will raise :exc:`OSError`. + :meth:`!write` and :meth:`truncate` will raise :exc:`OSError`. .. method:: writelines(lines, /) @@ -654,8 +653,9 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`read` (when called with a positive argument), :meth:`readinto` - and :meth:`write` methods on this class will only make one system call. + The :meth:`~RawIOBase.read` (when called with a positive argument), + :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this + class will only make one system call. A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with @@ -791,8 +791,8 @@ than raw I/O does. object under various conditions, including: * when the buffer gets too small for all pending data; - * when :meth:`flush()` is called; - * when a :meth:`seek()` is requested (for :class:`BufferedRandom` objects); + * when :meth:`flush` is called; + * when a :meth:`~IOBase.seek` is requested (for :class:`BufferedRandom` objects); * when the :class:`BufferedWriter` object is closed or destroyed. The constructor creates a :class:`BufferedWriter` for the given writeable @@ -826,8 +826,8 @@ than raw I/O does. :data:`DEFAULT_BUFFER_SIZE`. :class:`BufferedRandom` is capable of anything :class:`BufferedReader` or - :class:`BufferedWriter` can do. In addition, :meth:`seek` and :meth:`tell` - are guaranteed to be implemented. + :class:`BufferedWriter` can do. In addition, :meth:`~IOBase.seek` and + :meth:`~IOBase.tell` are guaranteed to be implemented. .. class:: BufferedRWPair(reader, writer, buffer_size=DEFAULT_BUFFER_SIZE, /) @@ -904,7 +904,7 @@ Text I/O .. method:: readline(size=-1, /) - Read until newline or EOF and return a single ``str``. If the stream is + Read until newline or EOF and return a single :class:`str`. If the stream is already at EOF, an empty string is returned. If *size* is specified, at most *size* characters will be read. @@ -913,22 +913,22 @@ Text I/O Change the stream position to the given *offset*. Behaviour depends on the *whence* parameter. The default value for *whence* is - :data:`SEEK_SET`. + :data:`!SEEK_SET`. - * :data:`SEEK_SET` or ``0``: seek from the start of the stream + * :data:`!SEEK_SET` or ``0``: seek from the start of the stream (the default); *offset* must either be a number returned by :meth:`TextIOBase.tell`, or zero. Any other *offset* value produces undefined behaviour. - * :data:`SEEK_CUR` or ``1``: "seek" to the current position; + * :data:`!SEEK_CUR` or ``1``: "seek" to the current position; *offset* must be zero, which is a no-operation (all other values are unsupported). - * :data:`SEEK_END` or ``2``: seek to the end of the stream; + * :data:`!SEEK_END` or ``2``: seek to the end of the stream; *offset* must be zero (all other values are unsupported). Return the new absolute position as an opaque number. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. method:: tell() @@ -988,10 +988,10 @@ Text I/O takes place. If *newline* is any of the other legal values, any ``'\n'`` characters written are translated to the given string. - If *line_buffering* is ``True``, :meth:`flush` is implied when a call to + If *line_buffering* is ``True``, :meth:`~IOBase.flush` is implied when a call to write contains a newline character or a carriage return. - If *write_through* is ``True``, calls to :meth:`write` are guaranteed + If *write_through* is ``True``, calls to :meth:`~BufferedIOBase.write` are guaranteed not to be buffered: any data written on the :class:`TextIOWrapper` object is immediately handled to its underlying binary *buffer*. @@ -1043,6 +1043,33 @@ Text I/O .. versionchanged:: 3.11 The method supports ``encoding="locale"`` option. + .. method:: seek(cookie, whence=os.SEEK_SET, /) + + Set the stream position. + Return the new stream position as an :class:`int`. + + Four operations are supported, + given by the following argument combinations: + + * ``seek(0, SEEK_SET)``: Rewind to the start of the stream. + * ``seek(cookie, SEEK_SET)``: Restore a previous position; + *cookie* **must be** a number returned by :meth:`tell`. + * ``seek(0, SEEK_END)``: Fast-forward to the end of the stream. + * ``seek(0, SEEK_CUR)``: Leave the current stream position unchanged. + + Any other argument combinations are invalid, + and may raise exceptions. + + .. seealso:: + + :data:`os.SEEK_SET`, :data:`os.SEEK_CUR`, and :data:`os.SEEK_END`. + + .. method:: tell() + + Return the stream position as an opaque number. + The return value of :meth:`!tell` can be given as input to :meth:`seek`, + to restore a previous stream position. + .. class:: StringIO(initial_value='', newline='\n') @@ -1070,7 +1097,7 @@ Text I/O .. method:: getvalue() - Return a ``str`` containing the entire contents of the buffer. + Return a :class:`str` containing the entire contents of the buffer. Newlines are decoded as if by :meth:`~TextIOBase.read`, although the stream position is not changed. @@ -1125,7 +1152,7 @@ Text I/O over a binary storage (such as a file) is significantly slower than binary I/O over the same storage, because it requires conversions between unicode and binary data using a character codec. This can become noticeable handling huge amounts of text data like large log files. Also, -:meth:`TextIOWrapper.tell` and :meth:`TextIOWrapper.seek` are both quite slow +:meth:`~TextIOBase.tell` and :meth:`~TextIOBase.seek` are both quite slow due to the reconstruction algorithm used. :class:`StringIO`, however, is a native in-memory unicode container and will @@ -1135,7 +1162,7 @@ Multi-threading ^^^^^^^^^^^^^^^ :class:`FileIO` objects are thread-safe to the extent that the operating system -calls (such as ``read(2)`` under Unix) they wrap are thread-safe too. +calls (such as :manpage:`read(2)` under Unix) they wrap are thread-safe too. Binary buffered objects (instances of :class:`BufferedReader`, :class:`BufferedWriter`, :class:`BufferedRandom` and :class:`BufferedRWPair`) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 9c2dff55703273..1de36b643c4dca 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -219,6 +219,13 @@ write code that handles both IP versions correctly. Address objects are ``True`` if the address is reserved for link-local usage. See :RFC:`3927`. + .. attribute:: ipv6_mapped + + :class:`IPv4Address` object representing the IPv4-mapped IPv6 address. See :RFC:`4291`. + + .. versionadded:: 3.13 + + .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index a0d794017e2602..8a4254cf15ebe2 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -41,7 +41,7 @@ operator can be mapped across two vectors to form an efficient dot-product: ================== ================= ================================================= ========================================= Iterator Arguments Results Example ================== ================= ================================================= ========================================= -:func:`count` start, [step] start, start+step, start+2*step, ... ``count(10) --> 10 11 12 13 14 ...`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) --> 10 11 12 13 14 ...`` :func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') --> A B C D A B C D ...`` :func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) --> 10 10 10`` ================== ================= ================================================= ========================================= @@ -147,10 +147,10 @@ loops that truncate the stream. >>> list(accumulate(data, max)) # running maximum [3, 4, 6, 6, 6, 9, 9, 9, 9, 9] - # Amortize a 5% loan of 1000 with 4 annual payments of 90 - >>> cashflows = [1000, -90, -90, -90, -90] - >>> list(accumulate(cashflows, lambda bal, pmt: bal*1.05 + pmt)) - [1000, 960.0, 918.0, 873.9000000000001, 827.5950000000001] + # Amortize a 5% loan of 1000 with 10 annual payments of 90 + >>> account_update = lambda bal, pmt: round(bal * 1.05) + pmt + >>> list(accumulate(repeat(-90, 10), account_update, initial=1_000)) + [1000, 960, 918, 874, 828, 779, 728, 674, 618, 559, 497] See :func:`functools.reduce` for a similar function that returns only the final accumulated value. @@ -798,10 +798,10 @@ which incur interpreter overhead. "Return first n items of the iterable as a list" return list(islice(iterable, n)) - def prepend(value, iterator): - "Prepend a single value in front of an iterator" + def prepend(value, iterable): + "Prepend a single value in front of an iterable" # prepend(1, [2, 3, 4]) --> 1 2 3 4 - return chain([value], iterator) + return chain([value], iterable) def tabulate(function, start=0): "Return function(0), function(1), ..." @@ -844,7 +844,7 @@ which incur interpreter overhead. return next(islice(iterable, n, None), default) def quantify(iterable, pred=bool): - "Count how many times the predicate is True" + "Given a predicate that returns True or False, count the True results." return sum(map(pred, iterable)) def all_equal(iterable): @@ -865,26 +865,23 @@ which incur interpreter overhead. # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x return next(filter(pred, iterable), default) - def iter_index(iterable, value, start=0): + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') --> 0 1 4 7 - try: - seq_index = iterable.index - except AttributeError: + seq_index = getattr(iterable, 'index', None) + if seq_index is None: # Slow path for general iterables - it = islice(iterable, start, None) - i = start - 1 - try: - while True: - yield (i := i + operator.indexOf(it, value) + 1) - except ValueError: - pass + it = islice(iterable, start, stop) + for i, element in enumerate(it, start): + if element is value or element == value: + yield i else: # Fast path for sequences + stop = len(iterable) if stop is None else stop i = start - 1 try: while True: - yield (i := seq_index(value, i+1)) + yield (i := seq_index(value, i+1, stop)) except ValueError: pass @@ -919,9 +916,9 @@ which incur interpreter overhead. args = [iter(iterable)] * n if incomplete == 'fill': return zip_longest(*args, fillvalue=fillvalue) - if incomplete == 'strict': + elif incomplete == 'strict': return zip(*args, strict=True) - if incomplete == 'ignore': + elif incomplete == 'ignore': return zip(*args) else: raise ValueError('Expected fill, strict, or ignore') @@ -929,9 +926,7 @@ which incur interpreter overhead. def sliding_window(iterable, n): # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG it = iter(iterable) - window = collections.deque(islice(it, n), maxlen=n) - if len(window) == n: - yield tuple(window) + window = collections.deque(islice(it, n-1), maxlen=n) for x in it: window.append(x) yield tuple(window) @@ -951,7 +946,10 @@ which incur interpreter overhead. nexts = cycle(islice(nexts, num_active)) def partition(pred, iterable): - "Use a predicate to partition entries into false entries and true entries" + """Partition entries into false entries and true entries. + + If *pred* is slow, consider wrapping it with functools.lru_cache(). + """ # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 t1, t2 = tee(iterable) return filterfalse(pred, t1), filter(pred, t2) @@ -1030,32 +1028,6 @@ The following recipes have a more mathematical flavor: s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - def sieve(n): - "Primes less than n" - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 - data = bytearray((0, 1)) * (n // 2) - data[:3] = 0, 0, 0 - limit = math.isqrt(n) + 1 - for p in compress(range(limit), data): - data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - data[2] = 1 - return iter_index(data, 1) if n > 2 else iter([]) - - def factor(n): - "Prime factors of n." - # factor(99) --> 3 3 11 - for prime in sieve(math.isqrt(n) + 1): - while True: - quotient, remainder = divmod(n, prime) - if remainder: - break - yield prime - n = quotient - if n == 1: - return - if n > 1: - yield n - def sum_of_squares(it): "Add up the squares of the input values." # sum_of_squares([10, 20, 30]) -> 1400 @@ -1068,24 +1040,31 @@ The following recipes have a more mathematical flavor: def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]]) --> (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) --> (49, 80), (41, 60) n = len(m2[0]) return batched(starmap(math.sumprod, product(m1, transpose(m2))), n) def convolve(signal, kernel): - """Linear convolution of two iterables. + """Discrete linear convolution of two iterables. + + The kernel is fully consumed before the calculations begin. + The signal is consumed lazily and can be infinite. + + Convolutions are mathematically commutative. + If the signal and kernel are swapped, + the output will be the same. Article: https://betterexplained.com/articles/intuitive-convolution/ Video: https://www.youtube.com/watch?v=KuXjwB4LzSA """ # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) - # convolve(data, [1, -1]) --> 1st finite difference (1st derivative) - # convolve(data, [1, -2, 1]) --> 2nd finite difference (2nd derivative) + # convolve(data, [1/2, 0, -1/2]) --> 1st derivative estimate + # convolve(data, [1, -2, 1]) --> 2nd derivative estimate kernel = tuple(kernel)[::-1] n = len(kernel) padded_signal = chain(repeat(0, n-1), signal, repeat(0, n-1)) - for window in sliding_window(padded_signal, n): - yield math.sumprod(kernel, window) + windowed_signal = sliding_window(padded_signal, n) + return map(math.sumprod, repeat(kernel), windowed_signal) def polynomial_from_roots(roots): """Compute a polynomial's coefficients from its roots. @@ -1104,11 +1083,50 @@ The following recipes have a more mathematical flavor: # Evaluate x³ -4x² -17x + 60 at x = 2.5 # polynomial_eval([1, -4, -17, 60], x=2.5) --> 8.125 n = len(coefficients) - if n == 0: - return x * 0 # coerce zero to the type of x + if not n: + return type(x)(0) powers = map(pow, repeat(x), reversed(range(n))) return math.sumprod(coefficients, powers) + def polynomial_derivative(coefficients): + """Compute the first derivative of a polynomial. + + f(x) = x³ -4x² -17x + 60 + f'(x) = 3x² -8x -17 + """ + # polynomial_derivative([1, -4, -17, 60]) -> [3, -8, -17] + n = len(coefficients) + powers = reversed(range(1, n)) + return list(map(operator.mul, coefficients, powers)) + + def sieve(n): + "Primes less than n." + # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + if n > 2: + yield 2 + start = 3 + data = bytearray((0, 1)) * (n // 2) + limit = math.isqrt(n) + 1 + for p in iter_index(data, 1, start, limit): + yield from iter_index(data, 1, start, p*p) + data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) + start = p*p + yield from iter_index(data, 1, start) + + def factor(n): + "Prime factors of n." + # factor(99) --> 3 3 11 + # factor(1_000_000_000_000_007) --> 47 59 360620266859 + # factor(1_000_000_000_000_403) --> 1000000000000403 + for prime in sieve(math.isqrt(n) + 1): + while not n % prime: + yield prime + n //= prime + if n == 1: + return + if n > 1: + yield n + def nth_combination(iterable, r, index): "Equivalent to list(combinations(iterable, r))[index]" pool = tuple(iterable) @@ -1284,7 +1302,7 @@ The following recipes have a more mathematical flavor: >>> polynomial_eval([], Fraction(2, 3)) Fraction(0, 1) >>> polynomial_eval([], Decimal('1.75')) - Decimal('0.00') + Decimal('0') >>> polynomial_eval([11], 7) == 11 True >>> polynomial_eval([11, 2], 7) == 11 * 7 + 2 @@ -1297,6 +1315,9 @@ The following recipes have a more mathematical flavor: >>> all(factored(x) == expanded(x) for x in range(-10, 11)) True + >>> polynomial_derivative([1, -4, -17, 60]) + [3, -8, -17] + >>> list(iter_index('AABCADEAF', 'A')) [0, 1, 4, 7] >>> list(iter_index('AABCADEAF', 'B')) @@ -1317,6 +1338,31 @@ The following recipes have a more mathematical flavor: [] >>> list(iter_index(iter('AABCADEAF'), 'A', 10)) [] + >>> list(iter_index('AABCADEAF', 'A', 1, 7)) + [1, 4] + >>> list(iter_index(iter('AABCADEAF'), 'A', 1, 7)) + [1, 4] + >>> # Verify that ValueErrors not swallowed (gh-107208) + >>> def assert_no_value(iterable, forbidden_value): + ... for item in iterable: + ... if item == forbidden_value: + ... raise ValueError + ... yield item + ... + >>> list(iter_index(assert_no_value('AABCADEAF', 'B'), 'A')) + Traceback (most recent call last): + ... + ValueError + >>> # Verify that both paths can find identical NaN values + >>> x = float('NaN') + >>> y = float('NaN') + >>> list(iter_index([0, x, x, y, 0], x)) + [1, 2] + >>> list(iter_index(iter([0, x, x, y, 0]), x)) + [1, 2] + >>> # Test list input. Lists do not support None for the stop argument + >>> list(iter_index(list('AABCADEAF'), 'A')) + [0, 1, 4, 7] >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] @@ -1337,6 +1383,12 @@ The following recipes have a more mathematical flavor: >>> set(sieve(10_000)).isdisjoint(carmichael) True + >>> list(factor(99)) # Code example 1 + [3, 3, 11] + >>> list(factor(1_000_000_000_000_007)) # Code example 2 + [47, 59, 360620266859] + >>> list(factor(1_000_000_000_000_403)) # Code example 3 + [1000000000000403] >>> list(factor(0)) [] >>> list(factor(1)) @@ -1403,8 +1455,34 @@ The following recipes have a more mathematical flavor: >>> list(grouper('abcdefg', n=3, incomplete='ignore')) [('a', 'b', 'c'), ('d', 'e', 'f')] + >>> list(sliding_window('ABCDEFG', 1)) + [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)] + >>> list(sliding_window('ABCDEFG', 2)) + [('A', 'B'), ('B', 'C'), ('C', 'D'), ('D', 'E'), ('E', 'F'), ('F', 'G')] + >>> list(sliding_window('ABCDEFG', 3)) + [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] >>> list(sliding_window('ABCDEFG', 4)) [('A', 'B', 'C', 'D'), ('B', 'C', 'D', 'E'), ('C', 'D', 'E', 'F'), ('D', 'E', 'F', 'G')] + >>> list(sliding_window('ABCDEFG', 5)) + [('A', 'B', 'C', 'D', 'E'), ('B', 'C', 'D', 'E', 'F'), ('C', 'D', 'E', 'F', 'G')] + >>> list(sliding_window('ABCDEFG', 6)) + [('A', 'B', 'C', 'D', 'E', 'F'), ('B', 'C', 'D', 'E', 'F', 'G')] + >>> list(sliding_window('ABCDEFG', 7)) + [('A', 'B', 'C', 'D', 'E', 'F', 'G')] + >>> list(sliding_window('ABCDEFG', 8)) + [] + >>> try: + ... list(sliding_window('ABCDEFG', -1)) + ... except ValueError: + ... 'zero or negative n not supported' + ... + 'zero or negative n not supported' + >>> try: + ... list(sliding_window('ABCDEFG', 0)) + ... except ValueError: + ... 'zero or negative n not supported' + ... + 'zero or negative n not supported' >>> list(roundrobin('abc', 'd', 'ef')) ['a', 'd', 'e', 'b', 'f', 'c'] diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 00f585124a86b3..0ce4b697145cb3 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -9,11 +9,6 @@ **Source code:** :source:`Lib/json/__init__.py` -.. testsetup:: * - - import json - from json import AttrDict - -------------- `JSON (JavaScript Object Notation) `_, specified by @@ -59,12 +54,23 @@ Compact encoding:: Pretty printing:: >>> import json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) + >>> print(json.dumps({'6': 7, '4': 5}, sort_keys=True, indent=4)) { "4": 5, "6": 7 } +Specializing JSON object encoding:: + + >>> import json + >>> def custom_json(obj): + ... if isinstance(obj, complex): + ... return {'__complex__': True, 'real': obj.real, 'imag': obj.imag} + ... raise TypeError(f'Cannot serialize object of {type(obj)}') + ... + >>> json.dumps(1 + 2j, default=custom_json) + '{"__complex__": true, "real": 1.0, "imag": 2.0}' + Decoding JSON:: >>> import json @@ -197,7 +203,7 @@ Basic Usage dictionaries will be sorted by key. To use a custom :class:`JSONEncoder` subclass (e.g. one that overrides the - :meth:`default` method to serialize additional types), specify it with the + :meth:`~JSONEncoder.default` method to serialize additional types), specify it with the *cls* kwarg; otherwise :class:`JSONEncoder` is used. .. versionchanged:: 3.6 @@ -427,7 +433,7 @@ Encoders and Decoders Added support for int- and float-derived Enum classes. To extend this to recognize other objects, subclass and implement a - :meth:`default` method with another method that returns a serializable object + :meth:`~JSONEncoder.default` method with another method that returns a serializable object for ``o`` if possible, otherwise it should call the superclass implementation (to raise :exc:`TypeError`). @@ -488,7 +494,7 @@ Encoders and Decoders :exc:`TypeError`). For example, to support arbitrary iterators, you could implement - :meth:`default` like this:: + :meth:`~JSONEncoder.default` like this:: def default(self, o): try: @@ -548,44 +554,6 @@ Exceptions .. versionadded:: 3.5 -.. class:: AttrDict(**kwargs) - AttrDict(mapping, **kwargs) - AttrDict(iterable, **kwargs) - - Subclass of :class:`dict` object that also supports attribute style dotted access. - - This class is intended for use with the :attr:`object_hook` in - :func:`json.load` and :func:`json.loads`:: - - .. doctest:: - - >>> json_string = '{"mercury": 88, "venus": 225, "earth": 365, "mars": 687}' - >>> orbital_period = json.loads(json_string, object_hook=AttrDict) - >>> orbital_period['earth'] # Dict style lookup - 365 - >>> orbital_period.earth # Attribute style lookup - 365 - >>> orbital_period.keys() # All dict methods are present - dict_keys(['mercury', 'venus', 'earth', 'mars']) - - Attribute style access only works for keys that are valid attribute - names. In contrast, dictionary style access works for all keys. For - example, ``d.two words`` contains a space and is not syntactically - valid Python, so ``d["two words"]`` should be used instead. - - If a key has the same name as a dictionary method, then a dictionary - lookup finds the key and an attribute lookup finds the method: - - .. doctest:: - - >>> d = AttrDict(items=50) - >>> d['items'] # Lookup the key - 50 - >>> d.items() # Call the method - dict_items([('items', 50)]) - - .. versionadded:: 3.12 - Standard Compliance and Interoperability ---------------------------------------- @@ -726,7 +694,7 @@ The :mod:`json.tool` module provides a simple command line interface to validate and pretty-print JSON objects. If the optional ``infile`` and ``outfile`` arguments are not -specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively: +specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: .. code-block:: shell-session @@ -746,7 +714,7 @@ specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively: Command line options ^^^^^^^^^^^^^^^^^^^^ -.. cmdoption:: infile +.. option:: infile The JSON file to be validated or pretty-printed: @@ -764,38 +732,38 @@ Command line options } ] - If *infile* is not specified, read from :attr:`sys.stdin`. + If *infile* is not specified, read from :data:`sys.stdin`. -.. cmdoption:: outfile +.. option:: outfile Write the output of the *infile* to the given *outfile*. Otherwise, write it - to :attr:`sys.stdout`. + to :data:`sys.stdout`. -.. cmdoption:: --sort-keys +.. option:: --sort-keys Sort the output of dictionaries alphabetically by key. .. versionadded:: 3.5 -.. cmdoption:: --no-ensure-ascii +.. option:: --no-ensure-ascii Disable escaping of non-ascii characters, see :func:`json.dumps` for more information. .. versionadded:: 3.9 -.. cmdoption:: --json-lines +.. option:: --json-lines Parse every input line as separate JSON object. .. versionadded:: 3.8 -.. cmdoption:: --indent, --tab, --no-indent, --compact +.. option:: --indent, --tab, --no-indent, --compact Mutually exclusive options for whitespace control. .. versionadded:: 3.9 -.. cmdoption:: -h, --help +.. option:: -h, --help Show the help message. diff --git a/Doc/library/kde_example.png b/Doc/library/kde_example.png new file mode 100644 index 00000000000000..f4504895699974 Binary files /dev/null and b/Doc/library/kde_example.png differ diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index f2abb3638a141f..0d48892fcdab97 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -303,7 +303,7 @@ The :mod:`locale` module defines the following exception and functions: *language code* and *encoding* may be ``None`` if their values cannot be determined. - .. deprecated-removed:: 3.11 3.13 + .. deprecated-removed:: 3.11 3.15 .. function:: getlocale(category=LC_CTYPE) @@ -370,16 +370,6 @@ The :mod:`locale` module defines the following exception and functions: encoding for the locale code just like :func:`setlocale`. -.. function:: resetlocale(category=LC_ALL) - - Sets the locale for *category* to the default setting. - - The default setting is determined by calling :func:`getdefaultlocale`. - *category* defaults to :const:`LC_ALL`. - - .. deprecated-removed:: 3.11 3.13 - - .. function:: strcoll(string1, string2) Compares two strings according to the current :const:`LC_COLLATE` setting. As @@ -464,11 +454,16 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_CTYPE - .. index:: pair: module; string + Locale category for the character type functions. Most importantly, this + category defines the text encoding, i.e. how bytes are interpreted as + Unicode codepoints. See :pep:`538` and :pep:`540` for how this variable + might be automatically coerced to ``C.UTF-8`` to avoid issues created by + invalid settings in containers or incompatible settings passed over remote + SSH connections. - Locale category for the character type functions. Depending on the settings of - this category, the functions of module :mod:`string` dealing with case change - their behaviour. + Python doesn't internally use locale-dependent character transformation functions + from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent + equivalents like :c:macro:`!Py_TOLOWER`. .. data:: LC_COLLATE diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 250246b5cd9adc..85a53e6aa7a78b 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -87,6 +87,10 @@ in :mod:`logging` itself) and defining handlers which are declared either in provides a mechanism to present the choices and load the chosen configuration). + It will raise :exc:`FileNotFoundError` if the file + doesn't exist and :exc:`RuntimeError` if the file is invalid or + empty. + :param fname: A filename, or a file-like object, or an instance derived from :class:`~configparser.RawConfigParser`. If a ``RawConfigParser``-derived instance is passed, it is used as @@ -111,7 +115,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in they or their ancestors are explicitly named in the logging configuration. - :param encoding: The encoding used to open file when *fname* is filename. + :param encoding: The encoding used to open file when *fname* is filename. .. versionchanged:: 3.4 An instance of a subclass of :class:`~configparser.RawConfigParser` is @@ -126,6 +130,10 @@ in :mod:`logging` itself) and defining handlers which are declared either in .. versionadded:: 3.10 The *encoding* parameter is added. + .. versionchanged:: 3.12 + An exception will be thrown if the provided file + doesn't exist or is invalid or empty. + .. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) Starts up a socket server on the specified port, and listens for new @@ -249,11 +257,11 @@ otherwise, the context is used to determine what to instantiate. which correspond to the arguments passed to create a :class:`~logging.Formatter` object: - * ``format`` - * ``datefmt`` - * ``style`` - * ``validate`` (since version >=3.8) - * ``defaults`` (since version >=3.12) + * ``format`` + * ``datefmt`` + * ``style`` + * ``validate`` (since version >=3.8) + * ``defaults`` (since version >=3.12) An optional ``class`` key indicates the name of the formatter's class (as a dotted module and class name). The instantiation @@ -536,9 +544,9 @@ valid keyword parameter name, and so will not clash with the names of the keyword arguments used in the call. The ``'()'`` also serves as a mnemonic that the corresponding value is a callable. - .. versionchanged:: 3.11 - The ``filters`` member of ``handlers`` and ``loggers`` can take - filter instances in addition to ids. +.. versionchanged:: 3.11 + The ``filters`` member of ``handlers`` and ``loggers`` can take + filter instances in addition to ids. You can also specify a special key ``'.'`` whose value is a dictionary is a mapping of attribute names to values. If found, the specified attributes will @@ -677,7 +685,8 @@ resolve to ``'dev_team@domain.tld'`` and the string ``'support_team@domain.tld'``. The ``subject`` value could be accessed using either ``'cfg://handlers.email.subject'`` or, equivalently, ``'cfg://handlers.email[subject]'``. The latter form only needs to be -used if the key contains spaces or non-alphanumeric characters. If an +used if the key contains spaces or non-alphanumeric characters. Please note +that the characters ``[`` and ``]`` are not allowed in the keys. If an index value consists only of decimal digits, access will be attempted using the corresponding integer value, falling back to the string value if needed. diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index d4429d3d0a4f73..2dd4bd081b0429 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -97,7 +97,7 @@ sends logging output to a disk file. It inherits the output functionality from Returns a new instance of the :class:`FileHandler` class. The specified file is opened and used as the stream for logging. If *mode* is not specified, - :const:`'a'` is used. If *encoding* is not ``None``, it is used to open the file + ``'a'`` is used. If *encoding* is not ``None``, it is used to open the file with that encoding. If *delay* is true, then file opening is deferred until the first call to :meth:`emit`. By default, the file grows indefinitely. If *errors* is specified, it's used to determine how encoding errors are handled. @@ -182,7 +182,7 @@ for this value. Returns a new instance of the :class:`WatchedFileHandler` class. The specified file is opened and used as the stream for logging. If *mode* is not specified, - :const:`'a'` is used. If *encoding* is not ``None``, it is used to open the file + ``'a'`` is used. If *encoding* is not ``None``, it is used to open the file with that encoding. If *delay* is true, then file opening is deferred until the first call to :meth:`emit`. By default, the file grows indefinitely. If *errors* is provided, it determines how encoding errors are handled. @@ -656,9 +656,7 @@ supports sending logging messages to a remote or local Unix syslog. to the other end. This method is called during handler initialization, but it's not regarded as an error if the other end isn't listening at this point - the method will be called again when emitting an event, if - but it's not regarded as an error if the other end isn't listening yet - --- the method will be called again when emitting an event, - if there is no socket at that point. + there is no socket at that point. .. versionadded:: 3.11 @@ -917,8 +915,9 @@ should, then :meth:`flush` is expected to do the flushing. .. method:: flush() - You can override this to implement custom flushing behavior. This version - just zaps the buffer to empty. + For a :class:`BufferingHandler` instance, flushing means that it sets the + buffer to an empty list. This method can be overwritten to implement more useful + flushing behavior. .. method:: shouldFlush(record) @@ -950,9 +949,9 @@ should, then :meth:`flush` is expected to do the flushing. .. method:: flush() - For a :class:`MemoryHandler`, flushing means just sending the buffered + For a :class:`MemoryHandler` instance, flushing means just sending the buffered records to the target, if there is one. The buffer is also cleared when - this happens. Override if you want different behavior. + buffered records are sent to the target. Override if you want different behavior. .. method:: setTarget(target) @@ -1051,8 +1050,8 @@ possible, while any potentially slow operations (such as sending an email via occur (e.g. because a bounded queue has filled up), the :meth:`~logging.Handler.handleError` method is called to handle the error. This can result in the record silently being dropped (if - :attr:`logging.raiseExceptions` is ``False``) or a message printed to - ``sys.stderr`` (if :attr:`logging.raiseExceptions` is ``True``). + :data:`logging.raiseExceptions` is ``False``) or a message printed to + ``sys.stderr`` (if :data:`logging.raiseExceptions` is ``True``). .. method:: prepare(record) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 22412e1a2113bb..acdeb88a546261 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -275,6 +275,10 @@ is the module's name in the Python package namespace. .. versionchanged:: 3.8 The *stacklevel* parameter was added. + .. versionchanged:: 3.13 + Remove the undocumented ``warn()`` method which was an alias to the + :meth:`warning` method. + .. method:: Logger.info(msg, *args, **kwargs) @@ -287,10 +291,6 @@ is the module's name in the Python package namespace. Logs a message with level :const:`WARNING` on this logger. The arguments are interpreted as for :meth:`debug`. - .. note:: There is an obsolete method ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - .. method:: Logger.error(msg, *args, **kwargs) Logs a message with level :const:`ERROR` on this logger. The arguments are @@ -397,21 +397,39 @@ have specific values relative to the predefined levels. If you define a level with the same numeric value, it overwrites the predefined value; the predefined name is lost. -+--------------+---------------+ -| Level | Numeric value | -+==============+===============+ -| ``CRITICAL`` | 50 | -+--------------+---------------+ -| ``ERROR`` | 40 | -+--------------+---------------+ -| ``WARNING`` | 30 | -+--------------+---------------+ -| ``INFO`` | 20 | -+--------------+---------------+ -| ``DEBUG`` | 10 | -+--------------+---------------+ -| ``NOTSET`` | 0 | -+--------------+---------------+ ++-----------------------+---------------+-------------------------------------+ +| Level | Numeric value | What it means / When to use it | ++=======================+===============+=====================================+ +| .. py:data:: NOTSET | 0 | When set on a logger, indicates that| +| | | ancestor loggers are to be consulted| +| | | to determine the effective level. | +| | | If that still resolves to | +| | | :const:`!NOTSET`, then all events | +| | | are logged. When set on a handler, | +| | | all events are handled. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: DEBUG | 10 | Detailed information, typically only| +| | | of interest to a developer trying to| +| | | diagnose a problem. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: INFO | 20 | Confirmation that things are working| +| | | as expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: WARNING | 30 | An indication that something | +| | | unexpected happened, or that a | +| | | problem might occur in the near | +| | | future (e.g. 'disk space low'). The | +| | | software is still working as | +| | | expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: ERROR | 40 | Due to a more serious problem, the | +| | | software has not been able to | +| | | perform some function. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: CRITICAL | 50 | A serious error, indicating that the| +| | | program itself may be unable to | +| | | continue running. | ++-----------------------+---------------+-------------------------------------+ .. _handler: @@ -421,7 +439,7 @@ Handler Objects Handlers have the following attributes and methods. Note that :class:`Handler` is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call +subclasses. However, the :meth:`!__init__` method in subclasses needs to call :meth:`Handler.__init__`. .. class:: Handler @@ -889,7 +907,7 @@ you want to use. In the case of {}-formatting, you can specify formatting flags by placing them after the attribute name, separated from it with a colon. For example: a -placeholder of ``{msecs:03d}`` would format a millisecond value of ``4`` as +placeholder of ``{msecs:03.0f}`` would format a millisecond value of ``4`` as ``004``. Refer to the :meth:`str.format` documentation for full details on the options available to you. @@ -984,10 +1002,14 @@ LoggerAdapter Objects information into logging calls. For a usage example, see the section on :ref:`adding contextual information to your logging output `. -.. class:: LoggerAdapter(logger, extra) +.. class:: LoggerAdapter(logger, extra, merge_extra=False) Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance and a dict-like object. + underlying :class:`Logger` instance, a dict-like object (*extra*), and a + boolean (*merge_extra*) indicating whether or not the *extra* argument of + individual log calls should be merged with the :class:`LoggerAdapter` extra. + The default behavior is to ignore the *extra* argument of individual log + calls and only use the one of the :class:`LoggerAdapter` instance .. method:: process(msg, kwargs) @@ -997,23 +1019,42 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. + .. attribute:: manager -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + Delegates to the underlying :attr:`!manager`` on *logger*. + + .. attribute:: _log + + Delegates to the underlying :meth:`!_log`` method on *logger*. + + In addition to the above, :class:`LoggerAdapter` supports the following + methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, + :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, + :meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, + :meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and + :meth:`~Logger.hasHandlers`. These methods have the same signatures as their + counterparts in :class:`Logger`, so you can use the two types of instances + interchangeably. + + .. versionchanged:: 3.2 -.. versionchanged:: 3.6 - Attribute :attr:`manager` and method :meth:`_log` were added, which - delegate to the underlying logger and allow adapters to be nested. + The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, + :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added + to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + + .. versionchanged:: 3.6 + + Attribute :attr:`!manager` and method :meth:`!_log` were added, which + delegate to the underlying logger and allow adapters to be nested. + + .. versionchanged:: 3.13 + + Remove the undocumented :meth:`!warn`` method which was an alias to the + :meth:`!warning` method. + + .. versionchanged:: 3.13 + + The *merge_extra* argument was added. Thread Safety @@ -1162,6 +1203,10 @@ functions. identical to ``warning``. As ``warn`` is deprecated, please do not use it - use ``warning`` instead. + .. versionchanged:: 3.13 + Remove the undocumented ``warn()`` function which was an alias to the + :func:`warning` function. + .. function:: error(msg, *args, **kwargs) @@ -1397,8 +1442,8 @@ functions. .. function:: setLoggerClass(klass) Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This + The class should define :meth:`!__init__` such that only a name argument is + required, and the :meth:`!__init__` should call :meth:`!Logger.__init__`. This function is typically called before any loggers are instantiated by applications which need to use custom logger behavior. After this call, as at any other time, do not instantiate loggers directly using the subclass: continue to use diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 868d4dcfb6c996..0d69c3bc01d1e2 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -100,7 +100,8 @@ Reading and writing compressed files *filters* arguments have the same meanings as for :class:`LZMACompressor`. :class:`LZMAFile` supports all the members specified by - :class:`io.BufferedIOBase`, except for :meth:`detach` and :meth:`truncate`. + :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach` + and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. The following method is also provided: @@ -332,19 +333,22 @@ the key ``"id"``, and may contain additional keys to specify filter-dependent options. Valid filter IDs are as follows: * Compression filters: - * :const:`FILTER_LZMA1` (for use with :const:`FORMAT_ALONE`) - * :const:`FILTER_LZMA2` (for use with :const:`FORMAT_XZ` and :const:`FORMAT_RAW`) + + * :const:`FILTER_LZMA1` (for use with :const:`FORMAT_ALONE`) + * :const:`FILTER_LZMA2` (for use with :const:`FORMAT_XZ` and :const:`FORMAT_RAW`) * Delta filter: - * :const:`FILTER_DELTA` + + * :const:`FILTER_DELTA` * Branch-Call-Jump (BCJ) filters: - * :const:`FILTER_X86` - * :const:`FILTER_IA64` - * :const:`FILTER_ARM` - * :const:`FILTER_ARMTHUMB` - * :const:`FILTER_POWERPC` - * :const:`FILTER_SPARC` + + * :const:`FILTER_X86` + * :const:`FILTER_IA64` + * :const:`FILTER_ARM` + * :const:`FILTER_ARMTHUMB` + * :const:`FILTER_POWERPC` + * :const:`FILTER_SPARC` A filter chain can consist of up to 4 filters, and cannot be empty. The last filter in the chain must be a compression filter, and any other filters must be @@ -353,21 +357,21 @@ delta or BCJ filters. Compression filters support the following options (specified as additional entries in the dictionary representing the filter): - * ``preset``: A compression preset to use as a source of default values for - options that are not specified explicitly. - * ``dict_size``: Dictionary size in bytes. This should be between 4 KiB and - 1.5 GiB (inclusive). - * ``lc``: Number of literal context bits. - * ``lp``: Number of literal position bits. The sum ``lc + lp`` must be at - most 4. - * ``pb``: Number of position bits; must be at most 4. - * ``mode``: :const:`MODE_FAST` or :const:`MODE_NORMAL`. - * ``nice_len``: What should be considered a "nice length" for a match. - This should be 273 or less. - * ``mf``: What match finder to use -- :const:`MF_HC3`, :const:`MF_HC4`, - :const:`MF_BT2`, :const:`MF_BT3`, or :const:`MF_BT4`. - * ``depth``: Maximum search depth used by match finder. 0 (default) means to - select automatically based on other filter options. +* ``preset``: A compression preset to use as a source of default values for + options that are not specified explicitly. +* ``dict_size``: Dictionary size in bytes. This should be between 4 KiB and + 1.5 GiB (inclusive). +* ``lc``: Number of literal context bits. +* ``lp``: Number of literal position bits. The sum ``lc + lp`` must be at + most 4. +* ``pb``: Number of position bits; must be at most 4. +* ``mode``: :const:`MODE_FAST` or :const:`MODE_NORMAL`. +* ``nice_len``: What should be considered a "nice length" for a match. + This should be 273 or less. +* ``mf``: What match finder to use -- :const:`MF_HC3`, :const:`MF_HC4`, + :const:`MF_BT2`, :const:`MF_BT3`, or :const:`MF_BT4`. +* ``depth``: Maximum search depth used by match finder. 0 (default) means to + select automatically based on other filter options. The delta filter stores the differences between bytes, producing more repetitive input for the compressor in certain circumstances. It supports one option, diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 56908dedea1b40..05ffaf6c9b336e 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -167,7 +167,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Return a representation of the message corresponding to *key*. If no such message exists, *default* is returned if the method was called as :meth:`get` and a :exc:`KeyError` exception is raised if the method was - called as :meth:`__getitem__`. The message is represented as an instance + called as :meth:`~object.__getitem__`. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified when the :class:`Mailbox` instance was initialized. @@ -424,6 +424,108 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. remove the underlying message while the returned file remains open. + .. method:: get_flags(key) + + Return as a string the flags that are set on the message + corresponding to *key*. + This is the same as ``get_message(key).get_flags()`` but much + faster, because it does not open the message file. + Use this method when iterating over the keys to determine which + messages are interesting to get. + + If you do have a :class:`MaildirMessage` object, use + its :meth:`~MaildirMessage.get_flags` method instead, because + changes made by the message's :meth:`~MaildirMessage.set_flags`, + :meth:`~MaildirMessage.add_flag` and :meth:`~MaildirMessage.remove_flag` + methods are not reflected here until the mailbox's + :meth:`__setitem__` method is called. + + .. versionadded:: 3.13 + + + .. method:: set_flags(key, flags) + + On the message corresponding to *key*, set the flags specified + by *flags* and unset all others. + Calling ``some_mailbox.set_flags(key, flags)`` is similar to :: + + one_message = some_mailbox.get_message(key) + one_message.set_flags(flags) + some_mailbox[key] = one_message + + but faster, because it does not open the message file. + + If you do have a :class:`MaildirMessage` object, use + its :meth:`~MaildirMessage.set_flags` method instead, because + changes made with this mailbox method will not be visible to the + message object's method, :meth:`~MaildirMessage.get_flags`. + + .. versionadded:: 3.13 + + + .. method:: add_flag(key, flag) + + On the message corresponding to *key*, set the flags specified + by *flag* without changing other flags. To add more than one + flag at a time, *flag* may be a string of more than one character. + + Considerations for using this method versus the message object's + :meth:`~MaildirMessage.add_flag` method are similar to + those for :meth:`set_flags`; see the discussion there. + + .. versionadded:: 3.13 + + + .. method:: remove_flag(key, flag) + + On the message corresponding to *key*, unset the flags specified + by *flag* without changing other flags. To remove more than one + flag at a time, *flag* may be a string of more than one character. + + Considerations for using this method versus the message object's + :meth:`~MaildirMessage.remove_flag` method are similar to + those for :meth:`set_flags`; see the discussion there. + + .. versionadded:: 3.13 + + + .. method:: get_info(key) + + Return a string containing the info for the message + corresponding to *key*. + This is the same as ``get_message(key).get_info()`` but much + faster, because it does not open the message file. + Use this method when iterating over the keys to determine which + messages are interesting to get. + + If you do have a :class:`MaildirMessage` object, use + its :meth:`~MaildirMessage.get_info` method instead, because + changes made by the message's :meth:`~MaildirMessage.set_info` method + are not reflected here until the mailbox's :meth:`__setitem__` method + is called. + + .. versionadded:: 3.13 + + + .. method:: set_info(key, info) + + Set the info of the message corresponding to *key* to *info*. + Calling ``some_mailbox.set_info(key, flags)`` is similar to :: + + one_message = some_mailbox.get_message(key) + one_message.set_info(info) + some_mailbox[key] = one_message + + but faster, because it does not open the message file. + + If you do have a :class:`MaildirMessage` object, use + its :meth:`~MaildirMessage.set_info` method instead, because + changes made with this mailbox method will not be visible to the + message object's method, :meth:`~MaildirMessage.get_info`. + + .. versionadded:: 3.13 + + .. seealso:: `maildir man page from Courier `_ @@ -477,7 +579,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. unlock() Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. + :c:func:`!flock` and :c:func:`!lockf` system calls. .. seealso:: @@ -588,7 +690,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. unlock() Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. For MH mailboxes, locking + :c:func:`!flock` and :c:func:`!lockf` system calls. For MH mailboxes, locking the mailbox means locking the :file:`.mh_sequences` file and, only for the duration of any operations that affect them, locking individual message files. @@ -686,7 +788,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. unlock() Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. + :c:func:`!flock` and :c:func:`!lockf` system calls. .. seealso:: @@ -737,7 +839,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. unlock() Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. + :c:func:`!flock` and :c:func:`!lockf` system calls. .. seealso:: @@ -838,7 +940,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. note:: A message is typically moved from :file:`new` to :file:`cur` after its - mailbox has been accessed, whether or not the message is has been + mailbox has been accessed, whether or not the message has been read. A message ``msg`` has been read if ``"S" in msg.get_flags()`` is ``True``. diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst deleted file mode 100644 index bfaedb46091991..00000000000000 --- a/Doc/library/mailcap.rst +++ /dev/null @@ -1,94 +0,0 @@ -:mod:`mailcap` --- Mailcap file handling -======================================== - -.. module:: mailcap - :synopsis: Mailcap file handling. - :deprecated: - -**Source code:** :source:`Lib/mailcap.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`mailcap` module is deprecated - (see :pep:`PEP 594 <594#mailcap>` for details). - The :mod:`mimetypes` module provides an alternative. - --------------- - -Mailcap files are used to configure how MIME-aware applications such as mail -readers and web browsers react to files with different MIME types. (The name -"mailcap" is derived from the phrase "mail capability".) For example, a mailcap -file might contain a line like ``video/mpeg; xmpeg %s``. Then, if the user -encounters an email message or web document with the MIME type -:mimetype:`video/mpeg`, ``%s`` will be replaced by a filename (usually one -belonging to a temporary file) and the :program:`xmpeg` program can be -automatically started to view the file. - -The mailcap format is documented in :rfc:`1524`, "A User Agent Configuration -Mechanism For Multimedia Mail Format Information", but is not an internet -standard. However, mailcap files are supported on most Unix systems. - - -.. function:: findmatch(caps, MIMEtype, key='view', filename='/dev/null', plist=[]) - - Return a 2-tuple; the first element is a string containing the command line to - be executed (which can be passed to :func:`os.system`), and the second element - is the mailcap entry for a given MIME type. If no matching MIME type can be - found, ``(None, None)`` is returned. - - *key* is the name of the field desired, which represents the type of activity to - be performed; the default value is 'view', since in the most common case you - simply want to view the body of the MIME-typed data. Other possible values - might be 'compose' and 'edit', if you wanted to create a new body of the given - MIME type or alter the existing body data. See :rfc:`1524` for a complete list - of these fields. - - *filename* is the filename to be substituted for ``%s`` in the command line; the - default value is ``'/dev/null'`` which is almost certainly not what you want, so - usually you'll override it by specifying a filename. - - *plist* can be a list containing named parameters; the default value is simply - an empty list. Each entry in the list must be a string containing the parameter - name, an equals sign (``'='``), and the parameter's value. Mailcap entries can - contain named parameters like ``%{foo}``, which will be replaced by the value - of the parameter named 'foo'. For example, if the command line ``showpartial - %{id} %{number} %{total}`` was in a mailcap file, and *plist* was set to - ``['id=1', 'number=2', 'total=3']``, the resulting command line would be - ``'showpartial 1 2 3'``. - - In a mailcap file, the "test" field can optionally be specified to test some - external condition (such as the machine architecture, or the window system in - use) to determine whether or not the mailcap line applies. :func:`findmatch` - will automatically check such conditions and skip the entry if the check fails. - - .. versionchanged:: 3.11 - - To prevent security issues with shell metacharacters (symbols that have - special effects in a shell command line), ``findmatch`` will refuse - to inject ASCII characters other than alphanumerics and ``@+=:,./-_`` - into the returned command line. - - If a disallowed character appears in *filename*, ``findmatch`` will always - return ``(None, None)`` as if no entry was found. - If such a character appears elsewhere (a value in *plist* or in *MIMEtype*), - ``findmatch`` will ignore all mailcap entries which use that value. - A :mod:`warning ` will be raised in either case. - -.. function:: getcaps() - - Returns a dictionary mapping MIME types to a list of mailcap file entries. This - dictionary must be passed to the :func:`findmatch` function. An entry is stored - as a list of dictionaries, but it shouldn't be necessary to know the details of - this representation. - - The information is derived from all of the mailcap files found on the system. - Settings in the user's mailcap file :file:`$HOME/.mailcap` will override - settings in the system mailcap files :file:`/etc/mailcap`, - :file:`/usr/etc/mailcap`, and :file:`/usr/local/etc/mailcap`. - -An example usage:: - - >>> import mailcap - >>> d = mailcap.getcaps() - >>> mailcap.findmatch(d, 'video/mpeg', filename='tmp1223') - ('xmpeg tmp1223', {'view': 'xmpeg %s'}) - diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 797f32408eac3d..9caf7230eed0aa 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -71,8 +71,8 @@ Number-theoretic and representation functions Return *n* factorial as an integer. Raises :exc:`ValueError` if *n* is not integral or is negative. - .. deprecated:: 3.9 - Accepting floats with integral values (like ``5.0``) is deprecated. + .. versionchanged:: 3.10 + Floats with integral values (like ``5.0``) are no longer accepted. .. function:: floor(x) @@ -224,11 +224,11 @@ Number-theoretic and representation functions of *x* and are floats. -.. function:: nextafter(x, y) +.. function:: nextafter(x, y, steps=1) - Return the next floating-point value after *x* towards *y*. + Return the floating-point value *steps* steps after *x* towards *y*. - If *x* is equal to *y*, return *y*. + If *x* is equal to *y*, return *y*, unless *steps* is zero. Examples: @@ -239,6 +239,9 @@ Number-theoretic and representation functions See also :func:`math.ulp`. + .. versionchanged:: 3.12 + Added the *steps* argument. + .. versionadded:: 3.9 .. function:: perm(n, k=None) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 69afadff1f5f42..3fa69e717329e4 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -19,7 +19,7 @@ the current file position, and :meth:`seek` through the file to different positi A memory-mapped file is created by the :class:`~mmap.mmap` constructor, which is different on Unix and on Windows. In either case you must provide a file descriptor for a file opened for update. If you wish to map an existing Python -file object, use its :meth:`fileno` method to obtain the correct value for the +file object, use its :meth:`~io.IOBase.fileno` method to obtain the correct value for the *fileno* parameter. Otherwise, you can open the file using the :func:`os.open` function, which returns a file descriptor directly (the file still needs to be closed when done). @@ -285,6 +285,14 @@ To map anonymous memory, -1 should be passed as the fileno along with the length values are ``os.SEEK_CUR`` or ``1`` (seek relative to the current position) and ``os.SEEK_END`` or ``2`` (seek relative to the file's end). + .. versionchanged:: 3.13 + Return the new absolute position instead of ``None``. + + .. method:: seekable() + + Return whether the file supports seeking, and the return value is always ``True``. + + .. versionadded:: 3.13 .. method:: size() diff --git a/Doc/library/msilib.rst b/Doc/library/msilib.rst deleted file mode 100644 index fbe55db9372300..00000000000000 --- a/Doc/library/msilib.rst +++ /dev/null @@ -1,565 +0,0 @@ -:mod:`msilib` --- Read and write Microsoft Installer files -========================================================== - -.. module:: msilib - :platform: Windows - :synopsis: Creation of Microsoft Installer files, and CAB files. - :deprecated: - -.. moduleauthor:: Martin v. Löwis -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/msilib/__init__.py` - -.. index:: single: msi - -.. deprecated-removed:: 3.11 3.13 - The :mod:`msilib` module is deprecated - (see :pep:`PEP 594 <594#msilib>` for details). - --------------- - -The :mod:`msilib` supports the creation of Microsoft Installer (``.msi``) files. -Because these files often contain an embedded "cabinet" file (``.cab``), it also -exposes an API to create CAB files. Support for reading ``.cab`` files is -currently not implemented; read support for the ``.msi`` database is possible. - -This package aims to provide complete access to all tables in an ``.msi`` file, -therefore, it is a fairly low-level API. One primary application of this -package is the creation of Python installer package itself (although that currently -uses a different version of ``msilib``). - -The package contents can be roughly split into four parts: low-level CAB -routines, low-level MSI routines, higher-level MSI routines, and standard table -structures. - - -.. function:: FCICreate(cabname, files) - - Create a new CAB file named *cabname*. *files* must be a list of tuples, each - containing the name of the file on disk, and the name of the file inside the CAB - file. - - The files are added to the CAB file in the order they appear in the list. All - files are added into a single CAB file, using the MSZIP compression algorithm. - - Callbacks to Python for the various steps of MSI creation are currently not - exposed. - - -.. function:: UuidCreate() - - Return the string representation of a new unique identifier. This wraps the - Windows API functions :c:func:`UuidCreate` and :c:func:`UuidToString`. - - -.. function:: OpenDatabase(path, persist) - - Return a new database object by calling MsiOpenDatabase. *path* is the file - name of the MSI file; *persist* can be one of the constants - ``MSIDBOPEN_CREATEDIRECT``, ``MSIDBOPEN_CREATE``, ``MSIDBOPEN_DIRECT``, - ``MSIDBOPEN_READONLY``, or ``MSIDBOPEN_TRANSACT``, and may include the flag - ``MSIDBOPEN_PATCHFILE``. See the Microsoft documentation for the meaning of - these flags; depending on the flags, an existing database is opened, or a new - one created. - - -.. function:: CreateRecord(count) - - Return a new record object by calling :c:func:`MSICreateRecord`. *count* is the - number of fields of the record. - - -.. function:: init_database(name, schema, ProductName, ProductCode, ProductVersion, Manufacturer) - - Create and return a new database *name*, initialize it with *schema*, and set - the properties *ProductName*, *ProductCode*, *ProductVersion*, and - *Manufacturer*. - - *schema* must be a module object containing ``tables`` and - ``_Validation_records`` attributes; typically, :mod:`msilib.schema` should be - used. - - The database will contain just the schema and the validation records when this - function returns. - - -.. function:: add_data(database, table, records) - - Add all *records* to the table named *table* in *database*. - - The *table* argument must be one of the predefined tables in the MSI schema, - e.g. ``'Feature'``, ``'File'``, ``'Component'``, ``'Dialog'``, ``'Control'``, - etc. - - *records* should be a list of tuples, each one containing all fields of a - record according to the schema of the table. For optional fields, - ``None`` can be passed. - - Field values can be ints, strings, or instances of the Binary class. - - -.. class:: Binary(filename) - - Represents entries in the Binary table; inserting such an object using - :func:`add_data` reads the file named *filename* into the table. - - -.. function:: add_tables(database, module) - - Add all table content from *module* to *database*. *module* must contain an - attribute *tables* listing all tables for which content should be added, and one - attribute per table that has the actual content. - - This is typically used to install the sequence tables. - - -.. function:: add_stream(database, name, path) - - Add the file *path* into the ``_Stream`` table of *database*, with the stream - name *name*. - - -.. function:: gen_uuid() - - Return a new UUID, in the format that MSI typically requires (i.e. in curly - braces, and with all hexdigits in uppercase). - - -.. seealso:: - - `FCICreate `_ - `UuidCreate `_ - `UuidToString `_ - -.. _database-objects: - -Database Objects ----------------- - - -.. method:: Database.OpenView(sql) - - Return a view object, by calling :c:func:`MSIDatabaseOpenView`. *sql* is the SQL - statement to execute. - - -.. method:: Database.Commit() - - Commit the changes pending in the current transaction, by calling - :c:func:`MSIDatabaseCommit`. - - -.. method:: Database.GetSummaryInformation(count) - - Return a new summary information object, by calling - :c:func:`MsiGetSummaryInformation`. *count* is the maximum number of updated - values. - -.. method:: Database.Close() - - Close the database object, through :c:func:`MsiCloseHandle`. - - .. versionadded:: 3.7 - -.. seealso:: - - `MSIDatabaseOpenView `_ - `MSIDatabaseCommit `_ - `MSIGetSummaryInformation `_ - `MsiCloseHandle `_ - -.. _view-objects: - -View Objects ------------- - - -.. method:: View.Execute(params) - - Execute the SQL query of the view, through :c:func:`MSIViewExecute`. If - *params* is not ``None``, it is a record describing actual values of the - parameter tokens in the query. - - -.. method:: View.GetColumnInfo(kind) - - Return a record describing the columns of the view, through calling - :c:func:`MsiViewGetColumnInfo`. *kind* can be either ``MSICOLINFO_NAMES`` or - ``MSICOLINFO_TYPES``. - - -.. method:: View.Fetch() - - Return a result record of the query, through calling :c:func:`MsiViewFetch`. - - -.. method:: View.Modify(kind, data) - - Modify the view, by calling :c:func:`MsiViewModify`. *kind* can be one of - ``MSIMODIFY_SEEK``, ``MSIMODIFY_REFRESH``, ``MSIMODIFY_INSERT``, - ``MSIMODIFY_UPDATE``, ``MSIMODIFY_ASSIGN``, ``MSIMODIFY_REPLACE``, - ``MSIMODIFY_MERGE``, ``MSIMODIFY_DELETE``, ``MSIMODIFY_INSERT_TEMPORARY``, - ``MSIMODIFY_VALIDATE``, ``MSIMODIFY_VALIDATE_NEW``, - ``MSIMODIFY_VALIDATE_FIELD``, or ``MSIMODIFY_VALIDATE_DELETE``. - - *data* must be a record describing the new data. - - -.. method:: View.Close() - - Close the view, through :c:func:`MsiViewClose`. - - -.. seealso:: - - `MsiViewExecute `_ - `MSIViewGetColumnInfo `_ - `MsiViewFetch `_ - `MsiViewModify `_ - `MsiViewClose `_ - -.. _summary-objects: - -Summary Information Objects ---------------------------- - - -.. method:: SummaryInformation.GetProperty(field) - - Return a property of the summary, through :c:func:`MsiSummaryInfoGetProperty`. - *field* is the name of the property, and can be one of the constants - ``PID_CODEPAGE``, ``PID_TITLE``, ``PID_SUBJECT``, ``PID_AUTHOR``, - ``PID_KEYWORDS``, ``PID_COMMENTS``, ``PID_TEMPLATE``, ``PID_LASTAUTHOR``, - ``PID_REVNUMBER``, ``PID_LASTPRINTED``, ``PID_CREATE_DTM``, - ``PID_LASTSAVE_DTM``, ``PID_PAGECOUNT``, ``PID_WORDCOUNT``, ``PID_CHARCOUNT``, - ``PID_APPNAME``, or ``PID_SECURITY``. - - -.. method:: SummaryInformation.GetPropertyCount() - - Return the number of summary properties, through - :c:func:`MsiSummaryInfoGetPropertyCount`. - - -.. method:: SummaryInformation.SetProperty(field, value) - - Set a property through :c:func:`MsiSummaryInfoSetProperty`. *field* can have the - same values as in :meth:`GetProperty`, *value* is the new value of the property. - Possible value types are integer and string. - - -.. method:: SummaryInformation.Persist() - - Write the modified properties to the summary information stream, using - :c:func:`MsiSummaryInfoPersist`. - - -.. seealso:: - - `MsiSummaryInfoGetProperty `_ - `MsiSummaryInfoGetPropertyCount `_ - `MsiSummaryInfoSetProperty `_ - `MsiSummaryInfoPersist `_ - -.. _record-objects: - -Record Objects --------------- - - -.. method:: Record.GetFieldCount() - - Return the number of fields of the record, through - :c:func:`MsiRecordGetFieldCount`. - - -.. method:: Record.GetInteger(field) - - Return the value of *field* as an integer where possible. *field* must - be an integer. - - -.. method:: Record.GetString(field) - - Return the value of *field* as a string where possible. *field* must - be an integer. - - -.. method:: Record.SetString(field, value) - - Set *field* to *value* through :c:func:`MsiRecordSetString`. *field* must be an - integer; *value* a string. - - -.. method:: Record.SetStream(field, value) - - Set *field* to the contents of the file named *value*, through - :c:func:`MsiRecordSetStream`. *field* must be an integer; *value* a string. - - -.. method:: Record.SetInteger(field, value) - - Set *field* to *value* through :c:func:`MsiRecordSetInteger`. Both *field* and - *value* must be an integer. - - -.. method:: Record.ClearData() - - Set all fields of the record to 0, through :c:func:`MsiRecordClearData`. - - -.. seealso:: - - `MsiRecordGetFieldCount `_ - `MsiRecordSetString `_ - `MsiRecordSetStream `_ - `MsiRecordSetInteger `_ - `MsiRecordClearData `_ - -.. _msi-errors: - -Errors ------- - -All wrappers around MSI functions raise :exc:`MSIError`; the string inside the -exception will contain more detail. - - -.. _cab: - -CAB Objects ------------ - - -.. class:: CAB(name) - - The class :class:`CAB` represents a CAB file. During MSI construction, files - will be added simultaneously to the ``Files`` table, and to a CAB file. Then, - when all files have been added, the CAB file can be written, then added to the - MSI file. - - *name* is the name of the CAB file in the MSI file. - - - .. method:: append(full, file, logical) - - Add the file with the pathname *full* to the CAB file, under the name - *logical*. If there is already a file named *logical*, a new file name is - created. - - Return the index of the file in the CAB file, and the new name of the file - inside the CAB file. - - - .. method:: commit(database) - - Generate a CAB file, add it as a stream to the MSI file, put it into the - ``Media`` table, and remove the generated file from the disk. - - -.. _msi-directory: - -Directory Objects ------------------ - - -.. class:: Directory(database, cab, basedir, physical, logical, default, [componentflags]) - - Create a new directory in the Directory table. There is a current component at - each point in time for the directory, which is either explicitly created through - :meth:`start_component`, or implicitly when files are added for the first time. - Files are added into the current component, and into the cab file. To create a - directory, a base directory object needs to be specified (can be ``None``), the - path to the physical directory, and a logical directory name. *default* - specifies the DefaultDir slot in the directory table. *componentflags* specifies - the default flags that new components get. - - - .. method:: start_component(component=None, feature=None, flags=None, keyfile=None, uuid=None) - - Add an entry to the Component table, and make this component the current - component for this directory. If no component name is given, the directory - name is used. If no *feature* is given, the current feature is used. If no - *flags* are given, the directory's default flags are used. If no *keyfile* - is given, the KeyPath is left null in the Component table. - - - .. method:: add_file(file, src=None, version=None, language=None) - - Add a file to the current component of the directory, starting a new one - if there is no current component. By default, the file name in the source - and the file table will be identical. If the *src* file is specified, it - is interpreted relative to the current directory. Optionally, a *version* - and a *language* can be specified for the entry in the File table. - - - .. method:: glob(pattern, exclude=None) - - Add a list of files to the current component as specified in the glob - pattern. Individual files can be excluded in the *exclude* list. - - - .. method:: remove_pyc() - - Remove ``.pyc`` files on uninstall. - - -.. seealso:: - - `Directory Table `_ - `File Table `_ - `Component Table `_ - `FeatureComponents Table `_ - -.. _features: - -Features --------- - - -.. class:: Feature(db, id, title, desc, display, level=1, parent=None, directory=None, attributes=0) - - Add a new record to the ``Feature`` table, using the values *id*, *parent.id*, - *title*, *desc*, *display*, *level*, *directory*, and *attributes*. The - resulting feature object can be passed to the :meth:`start_component` method of - :class:`Directory`. - - - .. method:: set_current() - - Make this feature the current feature of :mod:`msilib`. New components are - automatically added to the default feature, unless a feature is explicitly - specified. - - -.. seealso:: - - `Feature Table `_ - -.. _msi-gui: - -GUI classes ------------ - -:mod:`msilib` provides several classes that wrap the GUI tables in an MSI -database. However, no standard user interface is provided. - - -.. class:: Control(dlg, name) - - Base class of the dialog controls. *dlg* is the dialog object the control - belongs to, and *name* is the control's name. - - - .. method:: event(event, argument, condition=1, ordering=None) - - Make an entry into the ``ControlEvent`` table for this control. - - - .. method:: mapping(event, attribute) - - Make an entry into the ``EventMapping`` table for this control. - - - .. method:: condition(action, condition) - - Make an entry into the ``ControlCondition`` table for this control. - - -.. class:: RadioButtonGroup(dlg, name, property) - - Create a radio button control named *name*. *property* is the installer property - that gets set when a radio button is selected. - - - .. method:: add(name, x, y, width, height, text, value=None) - - Add a radio button named *name* to the group, at the coordinates *x*, *y*, - *width*, *height*, and with the label *text*. If *value* is ``None``, it - defaults to *name*. - - -.. class:: Dialog(db, name, x, y, w, h, attr, title, first, default, cancel) - - Return a new :class:`Dialog` object. An entry in the ``Dialog`` table is made, - with the specified coordinates, dialog attributes, title, name of the first, - default, and cancel controls. - - - .. method:: control(name, type, x, y, width, height, attributes, property, text, control_next, help) - - Return a new :class:`Control` object. An entry in the ``Control`` table is - made with the specified parameters. - - This is a generic method; for specific types, specialized methods are - provided. - - - .. method:: text(name, x, y, width, height, attributes, text) - - Add and return a ``Text`` control. - - - .. method:: bitmap(name, x, y, width, height, text) - - Add and return a ``Bitmap`` control. - - - .. method:: line(name, x, y, width, height) - - Add and return a ``Line`` control. - - - .. method:: pushbutton(name, x, y, width, height, attributes, text, next_control) - - Add and return a ``PushButton`` control. - - - .. method:: radiogroup(name, x, y, width, height, attributes, property, text, next_control) - - Add and return a ``RadioButtonGroup`` control. - - - .. method:: checkbox(name, x, y, width, height, attributes, property, text, next_control) - - Add and return a ``CheckBox`` control. - - -.. seealso:: - - `Dialog Table `_ - `Control Table `_ - `Control Types `_ - `ControlCondition Table `_ - `ControlEvent Table `_ - `EventMapping Table `_ - `RadioButton Table `_ - -.. _msi-tables: - -Precomputed tables ------------------- - -:mod:`msilib` provides a few subpackages that contain only schema and table -definitions. Currently, these definitions are based on MSI version 2.0. - - -.. data:: schema - - This is the standard MSI schema for MSI 2.0, with the *tables* variable - providing a list of table definitions, and *_Validation_records* providing the - data for MSI validation. - - -.. data:: sequence - - This module contains table contents for the standard sequence tables: - *AdminExecuteSequence*, *AdminUISequence*, *AdvtExecuteSequence*, - *InstallExecuteSequence*, and *InstallUISequence*. - - -.. data:: text - - This module contains definitions for the UIText and ActionText tables, for the - standard installer actions. diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 42fffee6a0f449..0b059e746c61af 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -10,8 +10,8 @@ -------------- These functions provide access to some useful capabilities on Windows platforms. -Some higher-level modules use these functions to build the Windows -implementations of their services. For example, the :mod:`getpass` module uses +Some higher-level modules use these functions to build the Windows +implementations of their services. For example, the :mod:`getpass` module uses this in the implementation of the :func:`getpass` function. Further documentation on these functions can be found in the Platform API @@ -35,11 +35,11 @@ File Operations .. function:: locking(fd, mode, nbytes) - Lock part of a file based on file descriptor *fd* from the C runtime. Raises - :exc:`OSError` on failure. The locked region of the file extends from the + Lock part of a file based on file descriptor *fd* from the C runtime. Raises + :exc:`OSError` on failure. The locked region of the file extends from the current file position for *nbytes* bytes, and may continue beyond the end of the - file. *mode* must be one of the :const:`LK_\*` constants listed below. Multiple - regions in a file may be locked at the same time, but may not overlap. Adjacent + file. *mode* must be one of the :const:`!LK_\*` constants listed below. Multiple + regions in a file may be locked at the same time, but may not overlap. Adjacent regions are not merged; they must be unlocked individually. .. audit-event:: msvcrt.locking fd,mode,nbytes msvcrt.locking @@ -49,7 +49,7 @@ File Operations LK_RLCK Locks the specified bytes. If the bytes cannot be locked, the program - immediately tries again after 1 second. If, after 10 attempts, the bytes cannot + immediately tries again after 1 second. If, after 10 attempts, the bytes cannot be locked, :exc:`OSError` is raised. @@ -74,9 +74,9 @@ File Operations .. function:: open_osfhandle(handle, flags) - Create a C runtime file descriptor from the file handle *handle*. The *flags* + Create a C runtime file descriptor from the file handle *handle*. The *flags* parameter should be a bitwise OR of :const:`os.O_APPEND`, :const:`os.O_RDONLY`, - and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter + and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter to :func:`os.fdopen` to create a file object. .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle @@ -84,7 +84,7 @@ File Operations .. function:: get_osfhandle(fd) - Return the file handle for the file descriptor *fd*. Raises :exc:`OSError` if + Return the file handle for the file descriptor *fd*. Raises :exc:`OSError` if *fd* is not recognized. .. audit-event:: msvcrt.get_osfhandle fd msvcrt.get_osfhandle @@ -98,13 +98,14 @@ Console I/O .. function:: kbhit() - Return ``True`` if a keypress is waiting to be read. + Returns a nonzero value if a keypress is waiting to be read. Otherwise, + return 0. .. function:: getch() Read a keypress and return the resulting character as a byte string. - Nothing is echoed to the console. This call will block if a keypress + Nothing is echoed to the console. This call will block if a keypress is not already available, but will not wait for :kbd:`Enter` to be pressed. If the pressed key was a special function key, this will return ``'\000'`` or ``'\xe0'``; the next call will return the keycode. @@ -118,7 +119,7 @@ Console I/O .. function:: getche() - Similar to :func:`getch`, but the keypress will be echoed if it represents a + Similar to :func:`getch`, but the keypress will be echoed if it represents a printable character. @@ -157,4 +158,93 @@ Other Functions .. function:: heapmin() Force the :c:func:`malloc` heap to clean itself up and return unused blocks to - the operating system. On failure, this raises :exc:`OSError`. + the operating system. On failure, this raises :exc:`OSError`. + + +.. function:: set_error_mode(mode) + + Changes the location where the C runtime writes an error message for an error + that might end the program. *mode* must be one of the :const:`!OUT_\*` + constants listed below or :const:`REPORT_ERRMODE`. Returns the old setting + or -1 if an error occurs. Only available in + :ref:`debug build of Python `. + + +.. data:: OUT_TO_DEFAULT + + Error sink is determined by the app's type. Only available in + :ref:`debug build of Python `. + + +.. data:: OUT_TO_STDERR + + Error sink is a standard error. Only available in + :ref:`debug build of Python `. + + +.. data:: OUT_TO_MSGBOX + + Error sink is a message box. Only available in + :ref:`debug build of Python `. + + +.. data:: REPORT_ERRMODE + + Report the current error mode value. Only available in + :ref:`debug build of Python `. + + +.. function:: CrtSetReportMode(type, mode) + + Specifies the destination or destinations for a specific report type + generated by :c:func:`!_CrtDbgReport` in the MS VC++ runtime. *type* must be + one of the :const:`!CRT_\*` constants listed below. *mode* must be one of the + :const:`!CRTDBG_\*` constants listed below. Only available in + :ref:`debug build of Python `. + + +.. function:: CrtSetReportFile(type, file) + + After you use :func:`CrtSetReportMode` to specify :const:`CRTDBG_MODE_FILE`, + you can specify the file handle to receive the message text. *type* must be + one of the :const:`!CRT_\*` constants listed below. *file* shuld be the file + handle your want specified. Only available in + :ref:`debug build of Python `. + + +.. data:: CRT_WARN + + Warnings, messages, and information that doesn't need immediate attention. + + +.. data:: CRT_ERROR + + Errors, unrecoverable problems, and issues that require immediate attention. + + +.. data:: CRT_ASSERT + + Assertion failures. + + +.. data:: CRTDBG_MODE_DEBUG + + Writes the message to the debugger's output window. + + +.. data:: CRTDBG_MODE_FILE + + Writes the message to a user-supplied file handle. :func:`CrtSetReportFile` + should be called to define the specific file or stream to use as + the destination. + + +.. data:: CRTDBG_MODE_WNDW + + Creates a message box to display the message along with the ``Abort``, + ``Retry``, and ``Ignore`` buttons. + + +.. data:: CRTDBG_REPORT_MODE + + Returns current *mode* for the specified *type*. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 8454296b815b41..789a84b02d59d2 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -131,6 +131,12 @@ to start a process. These *start methods* are Code that requires *fork* should explicitly specify that via :func:`get_context` or :func:`set_start_method`. + .. versionchanged:: 3.12 + If Python is able to detect that your process has multiple threads, the + :func:`os.fork` function that this start method calls internally will + raise a :exc:`DeprecationWarning`. Use a different start method. + See the :func:`os.fork` documentation for further explanation. + *forkserver* When the program starts and selects the *forkserver* start method, a server process is spawned. From then on, whenever a new process @@ -990,13 +996,20 @@ Miscellaneous This number is not equivalent to the number of CPUs the current process can use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` + :func:`os.process_cpu_count` (or ``len(os.sched_getaffinity(0))``). When the number of CPUs cannot be determined a :exc:`NotImplementedError` is raised. .. seealso:: :func:`os.cpu_count` + :func:`os.process_cpu_count` + + .. versionchanged:: 3.13 + + The return value can also be overridden using the + :option:`-X cpu_count <-X>` flag or :envvar:`PYTHON_CPU_COUNT` as this is + merely a wrapper around the :mod:`os` cpu count APIs. .. function:: current_process() @@ -2208,7 +2221,7 @@ with the :class:`Pool` class. callbacks and has a parallel map implementation. *processes* is the number of worker processes to use. If *processes* is - ``None`` then the number returned by :func:`os.cpu_count` is used. + ``None`` then the number returned by :func:`os.process_cpu_count` is used. If *initializer* is not ``None`` then each worker process will call ``initializer(*initargs)`` when it starts. @@ -2243,6 +2256,10 @@ with the :class:`Pool` class. .. versionadded:: 3.4 *context* + .. versionchanged:: 3.13 + *processes* uses :func:`os.process_cpu_count` by default, instead of + :func:`os.cpu_count`. + .. note:: Worker processes within a :class:`Pool` typically live for the complete @@ -2565,7 +2582,7 @@ multiple connections at the same time. **Windows**: An item in *object_list* must either be an integer handle which is waitable (according to the definition used by the documentation of the Win32 function ``WaitForMultipleObjects()``) - or it can be an object with a :meth:`fileno` method which returns a + or it can be an object with a :meth:`~io.IOBase.fileno` method which returns a socket handle or pipe handle. (Note that pipe handles and socket handles are **not** waitable handles.) @@ -2613,7 +2630,6 @@ server:: The following code uses :func:`~multiprocessing.connection.wait` to wait for messages from multiple processes at once:: - import time, random from multiprocessing import Process, Pipe, current_process from multiprocessing.connection import wait @@ -2707,7 +2723,7 @@ handler type) for messages from different processes to get mixed up. Returns the logger used by :mod:`multiprocessing`. If necessary, a new one will be created. - When first created the logger has level :data:`logging.NOTSET` and no + When first created the logger has level :const:`logging.NOTSET` and no default handler. Messages sent to this logger will not by default propagate to the root logger. @@ -2769,27 +2785,27 @@ worker threads rather than worker processes. :meth:`~multiprocessing.pool.Pool.terminate` manually. *processes* is the number of worker threads to use. If *processes* is - ``None`` then the number returned by :func:`os.cpu_count` is used. + ``None`` then the number returned by :func:`os.process_cpu_count` is used. If *initializer* is not ``None`` then each worker process will call ``initializer(*initargs)`` when it starts. Unlike :class:`Pool`, *maxtasksperchild* and *context* cannot be provided. - .. note:: + .. note:: - A :class:`ThreadPool` shares the same interface as :class:`Pool`, which - is designed around a pool of processes and predates the introduction of - the :class:`concurrent.futures` module. As such, it inherits some - operations that don't make sense for a pool backed by threads, and it - has its own type for representing the status of asynchronous jobs, - :class:`AsyncResult`, that is not understood by any other libraries. + A :class:`ThreadPool` shares the same interface as :class:`Pool`, which + is designed around a pool of processes and predates the introduction of + the :class:`concurrent.futures` module. As such, it inherits some + operations that don't make sense for a pool backed by threads, and it + has its own type for representing the status of asynchronous jobs, + :class:`AsyncResult`, that is not understood by any other libraries. - Users should generally prefer to use - :class:`concurrent.futures.ThreadPoolExecutor`, which has a simpler - interface that was designed around threads from the start, and which - returns :class:`concurrent.futures.Future` instances that are - compatible with many other libraries, including :mod:`asyncio`. + Users should generally prefer to use + :class:`concurrent.futures.ThreadPoolExecutor`, which has a simpler + interface that was designed around threads from the start, and which + returns :class:`concurrent.futures.Future` instances that are + compatible with many other libraries, including :mod:`asyncio`. .. _multiprocessing-programming: @@ -2988,7 +3004,7 @@ Global variables Safe importing of main module Make sure that the main module can be safely imported by a new Python - interpreter without causing unintended side effects (such a starting a new + interpreter without causing unintended side effects (such as starting a new process). For example, using the *spawn* or *forkserver* start method diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 76046b34610abe..671130d9b29fc0 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -36,7 +36,7 @@ or other communications requiring the serialization/deserialization and copying of data. -.. class:: SharedMemory(name=None, create=False, size=0) +.. class:: SharedMemory(name=None, create=False, size=0, *, track=True) Creates a new shared memory block or attaches to an existing shared memory block. Each shared memory block is assigned a unique name. @@ -64,26 +64,45 @@ copying of data. memory block may be larger or equal to the size requested. When attaching to an existing shared memory block, the ``size`` parameter is ignored. + *track*, when enabled, registers the shared memory block with a resource + tracker process on platforms where the OS does not do this automatically. + The resource tracker ensures proper cleanup of the shared memory even + if all other processes with access to the memory exit without doing so. + Python processes created from a common ancestor using :mod:`multiprocessing` + facilities share a single resource tracker process, and the lifetime of + shared memory segments is handled automatically among these processes. + Python processes created in any other way will receive their own + resource tracker when accessing shared memory with *track* enabled. + This will cause the shared memory to be deleted by the resource tracker + of the first process that terminates. + To avoid this issue, users of :mod:`subprocess` or standalone Python + processes should set *track* to ``False`` when there is already another + process in place that does the bookkeeping. + *track* is ignored on Windows, which has its own tracking and + automatically deletes shared memory when all handles to it have been closed. + + .. versionchanged:: 3.13 Added *track* parameter. + .. method:: close() - Closes access to the shared memory from this instance. In order to - ensure proper cleanup of resources, all instances should call - ``close()`` once the instance is no longer needed. Note that calling - ``close()`` does not cause the shared memory block itself to be - destroyed. + Closes the file descriptor/handle to the shared memory from this + instance. :meth:`close()` should be called once access to the shared + memory block from this instance is no longer needed. Depending + on operating system, the underlying memory may or may not be freed + even if all handles to it have been closed. To ensure proper cleanup, + use the :meth:`unlink()` method. .. method:: unlink() - Requests that the underlying shared memory block be destroyed. In - order to ensure proper cleanup of resources, ``unlink()`` should be - called once (and only once) across all processes which have need - for the shared memory block. After requesting its destruction, a - shared memory block may or may not be immediately destroyed and - this behavior may differ across platforms. Attempts to access data - inside the shared memory block after ``unlink()`` has been called may - result in memory access errors. Note: the last process relinquishing - its hold on a shared memory block may call ``unlink()`` and - :meth:`close()` in either order. + Deletes the underlying shared memory block. This should be called only + once per shared memory block regardless of the number of handles to it, + even in other processes. + :meth:`unlink()` and :meth:`close()` can be called in any order, but + trying to access data inside a shared memory block after :meth:`unlink()` + may result in memory access errors, depending on platform. + + This method has no effect on Windows, where the only way to delete a + shared memory block is to close all handles. .. attribute:: buf @@ -255,16 +274,17 @@ shared memory blocks created using that manager are all released when the :keyword:`with` statement's code block finishes execution. -.. class:: ShareableList(sequence=None, *, name=None) +.. class:: ShareableList(sequence=None, \*, name=None) Provides a mutable list-like object where all values stored within are stored in a shared memory block. This constrains storable values to - only the ``int``, ``float``, ``bool``, ``str`` (less than 10M bytes each), - ``bytes`` (less than 10M bytes each), and ``None`` built-in data types. - It also notably differs from the built-in ``list`` type in that these - lists can not change their overall length (i.e. no append, insert, etc.) - and do not support the dynamic creation of new :class:`ShareableList` - instances via slicing. + only the ``int`` (signed 64-bit), ``float``, ``bool``, ``str`` (less + than 10M bytes each when encoded as utf-8), ``bytes`` (less than 10M + bytes each), and ``None`` built-in data types. It also notably + differs from the built-in ``list`` type in that these lists can not + change their overall length (i.e. no append, insert, etc.) and do not + support the dynamic creation of new :class:`ShareableList` instances + via slicing. *sequence* is used in populating a new ``ShareableList`` full of values. Set to ``None`` to instead attach to an already existing @@ -275,6 +295,35 @@ shared memory blocks created using that manager are all released when the existing ``ShareableList``, specify its shared memory block's unique name while leaving ``sequence`` set to ``None``. + .. note:: + + A known issue exists for :class:`bytes` and :class:`str` values. + If they end with ``\x00`` nul bytes or characters, those may be + *silently stripped* when fetching them by index from the + :class:`ShareableList`. This ``.rstrip(b'\x00')`` behavior is + considered a bug and may go away in the future. See :gh:`106939`. + + For applications where rstripping of trailing nulls is a problem, + work around it by always unconditionally appending an extra non-0 + byte to the end of such values when storing and unconditionally + removing it when fetching: + + .. doctest:: + + >>> from multiprocessing import shared_memory + >>> nul_bug_demo = shared_memory.ShareableList(['?\x00', b'\x03\x02\x01\x00\x00\x00']) + >>> nul_bug_demo[0] + '?' + >>> nul_bug_demo[1] + b'\x03\x02\x01' + >>> nul_bug_demo.shm.unlink() + >>> padded = shared_memory.ShareableList(['?\x00\x07', b'\x03\x02\x01\x00\x00\x00\x07']) + >>> padded[0][:-1] + '?\x00' + >>> padded[1][:-1] + b'\x03\x02\x01\x00\x00\x00' + >>> padded.shm.unlink() + .. method:: count(value) Returns the number of occurrences of ``value``. diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 88265d9b9e9e93..c36e5cfecfc6a8 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -51,9 +51,19 @@ the Unix :program:`ftp` program and other FTP clients. Exception raised by the :class:`~netrc.netrc` class when syntactical errors are encountered in source text. Instances of this exception provide three - interesting attributes: :attr:`msg` is a textual explanation of the error, - :attr:`filename` is the name of the source file, and :attr:`lineno` gives the - line number on which the error was found. + interesting attributes: + + .. attribute:: msg + + Textual explanation of the error. + + .. attribute:: filename + + The name of the source file. + + .. attribute:: lineno + + The line number on which the error was found. .. _netrc-objects: diff --git a/Doc/library/nis.rst b/Doc/library/nis.rst deleted file mode 100644 index 3fa7916c37b6a5..00000000000000 --- a/Doc/library/nis.rst +++ /dev/null @@ -1,72 +0,0 @@ - -:mod:`nis` --- Interface to Sun's NIS (Yellow Pages) -==================================================== - -.. module:: nis - :platform: Unix - :synopsis: Interface to Sun's NIS (Yellow Pages) library. - :deprecated: - -.. moduleauthor:: Fred Gansevles -.. sectionauthor:: Moshe Zadka - -.. deprecated-removed:: 3.11 3.13 - The :mod:`nis` module is deprecated - (see :pep:`PEP 594 <594#nis>` for details). - --------------- - -The :mod:`nis` module gives a thin wrapper around the NIS library, useful for -central administration of several hosts. - -Because NIS exists only on Unix systems, this module is only available for Unix. - -.. include:: ../includes/wasm-notavail.rst - -The :mod:`nis` module defines the following functions: - - -.. function:: match(key, mapname, domain=default_domain) - - Return the match for *key* in map *mapname*, or raise an error - (:exc:`nis.error`) if there is none. Both should be strings, *key* is 8-bit - clean. Return value is an arbitrary array of bytes (may contain ``NULL`` and - other joys). - - Note that *mapname* is first checked if it is an alias to another name. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: cat(mapname, domain=default_domain) - - Return a dictionary mapping *key* to *value* such that ``match(key, - mapname)==value``. Note that both keys and values of the dictionary are - arbitrary arrays of bytes. - - Note that *mapname* is first checked if it is an alias to another name. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: maps(domain=default_domain) - - Return a list of all valid maps. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: get_default_domain() - - Return the system default NIS domain. - - -The :mod:`nis` module defines the following exception: - -.. exception:: error - - An error raised when a NIS function returns an error code. - diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst deleted file mode 100644 index 143e4e0c427f9a..00000000000000 --- a/Doc/library/nntplib.rst +++ /dev/null @@ -1,586 +0,0 @@ -:mod:`nntplib` --- NNTP protocol client -======================================= - -.. module:: nntplib - :synopsis: NNTP protocol client (requires sockets). - :deprecated: - -**Source code:** :source:`Lib/nntplib.py` - -.. index:: - pair: NNTP; protocol - single: Network News Transfer Protocol - -.. deprecated:: 3.11 - The :mod:`nntplib` module is deprecated (see :pep:`594` for details). - -.. testsetup:: - - import warnings - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import nntplib - -.. testcleanup:: - - try: - s.quit() - except NameError: - pass - import sys - # Force a warning if any other file imports nntplib - sys.modules.pop('nntplib') - --------------- - -This module defines the class :class:`NNTP` which implements the client side of -the Network News Transfer Protocol. It can be used to implement a news reader -or poster, or automated news processors. It is compatible with :rfc:`3977` -as well as the older :rfc:`977` and :rfc:`2980`. - -.. include:: ../includes/wasm-notavail.rst - -Here are two small examples of how it can be used. To list some statistics -about a newsgroup and print the subjects of the last 10 articles:: - - >>> s = nntplib.NNTP('news.gmane.io') - >>> resp, count, first, last, name = s.group('gmane.comp.python.committers') - >>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) - Group gmane.comp.python.committers has 1096 articles, range 1 to 1096 - >>> resp, overviews = s.over((last - 9, last)) - >>> for id, over in overviews: - ... print(id, nntplib.decode_header(over['subject'])) - ... - 1087 Re: Commit privileges for Åukasz Langa - 1088 Re: 3.2 alpha 2 freeze - 1089 Re: 3.2 alpha 2 freeze - 1090 Re: Commit privileges for Åukasz Langa - 1091 Re: Commit privileges for Åukasz Langa - 1092 Updated ssh key - 1093 Re: Updated ssh key - 1094 Re: Updated ssh key - 1095 Hello fellow committers! - 1096 Re: Hello fellow committers! - >>> s.quit() - '205 Bye!' - -To post an article from a binary file (this assumes that the article has valid -headers, and that you have right to post on the particular newsgroup):: - - >>> s = nntplib.NNTP('news.gmane.io') - >>> f = open('article.txt', 'rb') - >>> s.post(f) - '240 Article posted successfully.' - >>> s.quit() - '205 Bye!' - -The module itself defines the following classes: - - -.. class:: NNTP(host, port=119, user=None, password=None, readermode=None, usenetrc=False, [timeout]) - - Return a new :class:`NNTP` object, representing a connection - to the NNTP server running on host *host*, listening at port *port*. - An optional *timeout* can be specified for the socket connection. - If the optional *user* and *password* are provided, or if suitable - credentials are present in :file:`/.netrc` and the optional flag *usenetrc* - is true, the ``AUTHINFO USER`` and ``AUTHINFO PASS`` commands are used - to identify and authenticate the user to the server. If the optional - flag *readermode* is true, then a ``mode reader`` command is sent before - authentication is performed. Reader mode is sometimes necessary if you are - connecting to an NNTP server on the local machine and intend to call - reader-specific commands, such as ``group``. If you get unexpected - :exc:`NNTPPermanentError`\ s, you might need to set *readermode*. - The :class:`NNTP` class supports the :keyword:`with` statement to - unconditionally consume :exc:`OSError` exceptions and to close the NNTP - connection when done, e.g.: - - >>> from nntplib import NNTP - >>> with NNTP('news.gmane.io') as n: - ... n.group('gmane.comp.python.committers') - ... # doctest: +SKIP - ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers') - >>> - - .. audit-event:: nntplib.connect self,host,port nntplib.NNTP - - .. audit-event:: nntplib.putline self,line nntplib.NNTP - - All commands will raise an :ref:`auditing event ` - ``nntplib.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. - - .. versionchanged:: 3.2 - *usenetrc* is now ``False`` by default. - - .. versionchanged:: 3.3 - Support for the :keyword:`with` statement was added. - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - -.. class:: NNTP_SSL(host, port=563, user=None, password=None, ssl_context=None, readermode=None, usenetrc=False, [timeout]) - - Return a new :class:`NNTP_SSL` object, representing an encrypted - connection to the NNTP server running on host *host*, listening at - port *port*. :class:`NNTP_SSL` objects have the same methods as - :class:`NNTP` objects. If *port* is omitted, port 563 (NNTPS) is used. - *ssl_context* is also optional, and is a :class:`~ssl.SSLContext` object. - Please read :ref:`ssl-security` for best practices. - All other parameters behave the same as for :class:`NNTP`. - - Note that SSL-on-563 is discouraged per :rfc:`4642`, in favor of - STARTTLS as described below. However, some servers only support the - former. - - .. audit-event:: nntplib.connect self,host,port nntplib.NNTP_SSL - - .. audit-event:: nntplib.putline self,line nntplib.NNTP_SSL - - All commands will raise an :ref:`auditing event ` - ``nntplib.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - -.. exception:: NNTPError - - Derived from the standard exception :exc:`Exception`, this is the base - class for all exceptions raised by the :mod:`nntplib` module. Instances - of this class have the following attribute: - - .. attribute:: response - - The response of the server if available, as a :class:`str` object. - - -.. exception:: NNTPReplyError - - Exception raised when an unexpected reply is received from the server. - - -.. exception:: NNTPTemporaryError - - Exception raised when a response code in the range 400--499 is received. - - -.. exception:: NNTPPermanentError - - Exception raised when a response code in the range 500--599 is received. - - -.. exception:: NNTPProtocolError - - Exception raised when a reply is received from the server that does not begin - with a digit in the range 1--5. - - -.. exception:: NNTPDataError - - Exception raised when there is some error in the response data. - - -.. _nntp-objects: - -NNTP Objects ------------- - -When connected, :class:`NNTP` and :class:`NNTP_SSL` objects support the -following methods and attributes. - -Attributes -^^^^^^^^^^ - -.. attribute:: NNTP.nntp_version - - An integer representing the version of the NNTP protocol supported by the - server. In practice, this should be ``2`` for servers advertising - :rfc:`3977` compliance and ``1`` for others. - - .. versionadded:: 3.2 - -.. attribute:: NNTP.nntp_implementation - - A string describing the software name and version of the NNTP server, - or :const:`None` if not advertised by the server. - - .. versionadded:: 3.2 - -Methods -^^^^^^^ - -The *response* that is returned as the first item in the return tuple of almost -all methods is the server's response: a string beginning with a three-digit -code. If the server's response indicates an error, the method raises one of -the above exceptions. - -Many of the following methods take an optional keyword-only argument *file*. -When the *file* argument is supplied, it must be either a :term:`file object` -opened for binary writing, or the name of an on-disk file to be written to. -The method will then write any data returned by the server (except for the -response line and the terminating dot) to the file; any list of lines, -tuples or objects that the method normally returns will be empty. - -.. versionchanged:: 3.2 - Many of the following methods have been reworked and fixed, which makes - them incompatible with their 3.1 counterparts. - - -.. method:: NNTP.quit() - - Send a ``QUIT`` command and close the connection. Once this method has been - called, no other methods of the NNTP object should be called. - - -.. method:: NNTP.getwelcome() - - Return the welcome message sent by the server in reply to the initial - connection. (This message sometimes contains disclaimers or help information - that may be relevant to the user.) - - -.. method:: NNTP.getcapabilities() - - Return the :rfc:`3977` capabilities advertised by the server, as a - :class:`dict` instance mapping capability names to (possibly empty) lists - of values. On legacy servers which don't understand the ``CAPABILITIES`` - command, an empty dictionary is returned instead. - - >>> s = NNTP('news.gmane.io') - >>> 'POST' in s.getcapabilities() - True - - .. versionadded:: 3.2 - - -.. method:: NNTP.login(user=None, password=None, usenetrc=True) - - Send ``AUTHINFO`` commands with the user name and password. If *user* - and *password* are ``None`` and *usenetrc* is true, credentials from - ``~/.netrc`` will be used if possible. - - Unless intentionally delayed, login is normally performed during the - :class:`NNTP` object initialization and separately calling this function - is unnecessary. To force authentication to be delayed, you must not set - *user* or *password* when creating the object, and must set *usenetrc* to - False. - - .. versionadded:: 3.2 - - -.. method:: NNTP.starttls(context=None) - - Send a ``STARTTLS`` command. This will enable encryption on the NNTP - connection. The *context* argument is optional and should be a - :class:`ssl.SSLContext` object. Please read :ref:`ssl-security` for best - practices. - - Note that this may not be done after authentication information has - been transmitted, and authentication occurs by default if possible during a - :class:`NNTP` object initialization. See :meth:`NNTP.login` for information - on suppressing this behavior. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - -.. method:: NNTP.newgroups(date, *, file=None) - - Send a ``NEWGROUPS`` command. The *date* argument should be a - :class:`datetime.date` or :class:`datetime.datetime` object. - Return a pair ``(response, groups)`` where *groups* is a list representing - the groups that are new since the given *date*. If *file* is supplied, - though, then *groups* will be empty. - - >>> from datetime import date, timedelta - >>> resp, groups = s.newgroups(date.today() - timedelta(days=3)) - >>> len(groups) # doctest: +SKIP - 85 - >>> groups[0] # doctest: +SKIP - GroupInfo(group='gmane.network.tor.devel', last='4', first='1', flag='m') - - -.. method:: NNTP.newnews(group, date, *, file=None) - - Send a ``NEWNEWS`` command. Here, *group* is a group name or ``'*'``, and - *date* has the same meaning as for :meth:`newgroups`. Return a pair - ``(response, articles)`` where *articles* is a list of message ids. - - This command is frequently disabled by NNTP server administrators. - - -.. method:: NNTP.list(group_pattern=None, *, file=None) - - Send a ``LIST`` or ``LIST ACTIVE`` command. Return a pair - ``(response, list)`` where *list* is a list of tuples representing all - the groups available from this NNTP server, optionally matching the - pattern string *group_pattern*. Each tuple has the form - ``(group, last, first, flag)``, where *group* is a group name, *last* - and *first* are the last and first article numbers, and *flag* usually - takes one of these values: - - * ``y``: Local postings and articles from peers are allowed. - * ``m``: The group is moderated and all postings must be approved. - * ``n``: No local postings are allowed, only articles from peers. - * ``j``: Articles from peers are filed in the junk group instead. - * ``x``: No local postings, and articles from peers are ignored. - * ``=foo.bar``: Articles are filed in the ``foo.bar`` group instead. - - If *flag* has another value, then the status of the newsgroup should be - considered unknown. - - This command can return very large results, especially if *group_pattern* - is not specified. It is best to cache the results offline unless you - really need to refresh them. - - .. versionchanged:: 3.2 - *group_pattern* was added. - - -.. method:: NNTP.descriptions(grouppattern) - - Send a ``LIST NEWSGROUPS`` command, where *grouppattern* is a wildmat string as - specified in :rfc:`3977` (it's essentially the same as DOS or UNIX shell wildcard - strings). Return a pair ``(response, descriptions)``, where *descriptions* - is a dictionary mapping group names to textual descriptions. - - >>> resp, descs = s.descriptions('gmane.comp.python.*') - >>> len(descs) # doctest: +SKIP - 295 - >>> descs.popitem() # doctest: +SKIP - ('gmane.comp.python.bio.general', 'BioPython discussion list (Moderated)') - - -.. method:: NNTP.description(group) - - Get a description for a single group *group*. If more than one group matches - (if 'group' is a real wildmat string), return the first match. If no group - matches, return an empty string. - - This elides the response code from the server. If the response code is needed, - use :meth:`descriptions`. - - -.. method:: NNTP.group(name) - - Send a ``GROUP`` command, where *name* is the group name. The group is - selected as the current group, if it exists. Return a tuple - ``(response, count, first, last, name)`` where *count* is the (estimated) - number of articles in the group, *first* is the first article number in - the group, *last* is the last article number in the group, and *name* - is the group name. - - -.. method:: NNTP.over(message_spec, *, file=None) - - Send an ``OVER`` command, or an ``XOVER`` command on legacy servers. - *message_spec* can be either a string representing a message id, or - a ``(first, last)`` tuple of numbers indicating a range of articles in - the current group, or a ``(first, None)`` tuple indicating a range of - articles starting from *first* to the last article in the current group, - or :const:`None` to select the current article in the current group. - - Return a pair ``(response, overviews)``. *overviews* is a list of - ``(article_number, overview)`` tuples, one for each article selected - by *message_spec*. Each *overview* is a dictionary with the same number - of items, but this number depends on the server. These items are either - message headers (the key is then the lower-cased header name) or metadata - items (the key is then the metadata name prepended with ``":"``). The - following items are guaranteed to be present by the NNTP specification: - - * the ``subject``, ``from``, ``date``, ``message-id`` and ``references`` - headers - * the ``:bytes`` metadata: the number of bytes in the entire raw article - (including headers and body) - * the ``:lines`` metadata: the number of lines in the article body - - The value of each item is either a string, or :const:`None` if not present. - - It is advisable to use the :func:`decode_header` function on header - values when they may contain non-ASCII characters:: - - >>> _, _, first, last, _ = s.group('gmane.comp.python.devel') - >>> resp, overviews = s.over((last, last)) - >>> art_num, over = overviews[0] - >>> art_num - 117216 - >>> list(over.keys()) - ['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject'] - >>> over['from'] - '=?UTF-8?B?Ik1hcnRpbiB2LiBMw7Z3aXMi?= ' - >>> nntplib.decode_header(over['from']) - '"Martin v. Löwis" ' - - .. versionadded:: 3.2 - - -.. method:: NNTP.help(*, file=None) - - Send a ``HELP`` command. Return a pair ``(response, list)`` where *list* is a - list of help strings. - - -.. method:: NNTP.stat(message_spec=None) - - Send a ``STAT`` command, where *message_spec* is either a message id - (enclosed in ``'<'`` and ``'>'``) or an article number in the current group. - If *message_spec* is omitted or :const:`None`, the current article in the - current group is considered. Return a triple ``(response, number, id)`` - where *number* is the article number and *id* is the message id. - - >>> _, _, first, last, _ = s.group('gmane.comp.python.devel') - >>> resp, number, message_id = s.stat(first) - >>> number, message_id - (9099, '<20030112190404.GE29873@epoch.metaslash.com>') - - -.. method:: NNTP.next() - - Send a ``NEXT`` command. Return as for :meth:`.stat`. - - -.. method:: NNTP.last() - - Send a ``LAST`` command. Return as for :meth:`.stat`. - - -.. method:: NNTP.article(message_spec=None, *, file=None) - - Send an ``ARTICLE`` command, where *message_spec* has the same meaning as - for :meth:`.stat`. Return a tuple ``(response, info)`` where *info* - is a :class:`~collections.namedtuple` with three attributes *number*, - *message_id* and *lines* (in that order). *number* is the article number - in the group (or 0 if the information is not available), *message_id* the - message id as a string, and *lines* a list of lines (without terminating - newlines) comprising the raw message including headers and body. - - >>> resp, info = s.article('<20030112190404.GE29873@epoch.metaslash.com>') - >>> info.number - 0 - >>> info.message_id - '<20030112190404.GE29873@epoch.metaslash.com>' - >>> len(info.lines) - 65 - >>> info.lines[0] - b'Path: main.gmane.org!not-for-mail' - >>> info.lines[1] - b'From: Neal Norwitz ' - >>> info.lines[-3:] - [b'There is a patch for 2.3 as well as 2.2.', b'', b'Neal'] - - -.. method:: NNTP.head(message_spec=None, *, file=None) - - Same as :meth:`article()`, but sends a ``HEAD`` command. The *lines* - returned (or written to *file*) will only contain the message headers, not - the body. - - -.. method:: NNTP.body(message_spec=None, *, file=None) - - Same as :meth:`article()`, but sends a ``BODY`` command. The *lines* - returned (or written to *file*) will only contain the message body, not the - headers. - - -.. method:: NNTP.post(data) - - Post an article using the ``POST`` command. The *data* argument is either - a :term:`file object` opened for binary reading, or any iterable of bytes - objects (representing raw lines of the article to be posted). It should - represent a well-formed news article, including the required headers. The - :meth:`post` method automatically escapes lines beginning with ``.`` and - appends the termination line. - - If the method succeeds, the server's response is returned. If the server - refuses posting, a :class:`NNTPReplyError` is raised. - - -.. method:: NNTP.ihave(message_id, data) - - Send an ``IHAVE`` command. *message_id* is the id of the message to send - to the server (enclosed in ``'<'`` and ``'>'``). The *data* parameter - and the return value are the same as for :meth:`post()`. - - -.. method:: NNTP.date() - - Return a pair ``(response, date)``. *date* is a :class:`~datetime.datetime` - object containing the current date and time of the server. - - -.. method:: NNTP.slave() - - Send a ``SLAVE`` command. Return the server's *response*. - - -.. method:: NNTP.set_debuglevel(level) - - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request or response. A value of ``2`` or higher produces the maximum amount - of debugging output, logging each line sent and received on the connection - (including message text). - - -The following are optional NNTP extensions defined in :rfc:`2980`. Some of -them have been superseded by newer commands in :rfc:`3977`. - - -.. method:: NNTP.xhdr(hdr, str, *, file=None) - - Send an ``XHDR`` command. The *hdr* argument is a header keyword, e.g. - ``'subject'``. The *str* argument should have the form ``'first-last'`` - where *first* and *last* are the first and last article numbers to search. - Return a pair ``(response, list)``, where *list* is a list of pairs ``(id, - text)``, where *id* is an article number (as a string) and *text* is the text of - the requested header for that article. If the *file* parameter is supplied, then - the output of the ``XHDR`` command is stored in a file. If *file* is a string, - then the method will open a file with that name, write to it then close it. - If *file* is a :term:`file object`, then it will start calling :meth:`write` on - it to store the lines of the command output. If *file* is supplied, then the - returned *list* is an empty list. - - -.. method:: NNTP.xover(start, end, *, file=None) - - Send an ``XOVER`` command. *start* and *end* are article numbers - delimiting the range of articles to select. The return value is the - same of for :meth:`over()`. It is recommended to use :meth:`over()` - instead, since it will automatically use the newer ``OVER`` command - if available. - - -Utility functions ------------------ - -The module also defines the following utility function: - - -.. function:: decode_header(header_str) - - Decode a header value, un-escaping any escaped non-ASCII characters. - *header_str* must be a :class:`str` object. The unescaped value is - returned. Using this function is recommended to display some headers - in a human readable form:: - - >>> decode_header("Some subject") - 'Some subject' - >>> decode_header("=?ISO-8859-15?Q?D=E9buter_en_Python?=") - 'Débuter en Python' - >>> decode_header("Re: =?UTF-8?B?cHJvYmzDqG1lIGRlIG1hdHJpY2U=?=") - 'Re: problème de matrice' diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index b3dce151aee289..2a05b56db051f9 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -160,23 +160,23 @@ refer to ``MyIntegral`` and ``OtherTypeIKnowAbout`` as of :class:`Complex` (``a : A <: Complex``), and ``b : B <: Complex``. I'll consider ``a + b``: - 1. If ``A`` defines an :meth:`__add__` which accepts ``b``, all is - well. - 2. If ``A`` falls back to the boilerplate code, and it were to - return a value from :meth:`__add__`, we'd miss the possibility - that ``B`` defines a more intelligent :meth:`__radd__`, so the - boilerplate should return :const:`NotImplemented` from - :meth:`__add__`. (Or ``A`` may not implement :meth:`__add__` at - all.) - 3. Then ``B``'s :meth:`__radd__` gets a chance. If it accepts - ``a``, all is well. - 4. If it falls back to the boilerplate, there are no more possible - methods to try, so this is where the default implementation - should live. - 5. If ``B <: A``, Python tries ``B.__radd__`` before - ``A.__add__``. This is ok, because it was implemented with - knowledge of ``A``, so it can handle those instances before - delegating to :class:`Complex`. +1. If ``A`` defines an :meth:`__add__` which accepts ``b``, all is + well. +2. If ``A`` falls back to the boilerplate code, and it were to + return a value from :meth:`__add__`, we'd miss the possibility + that ``B`` defines a more intelligent :meth:`__radd__`, so the + boilerplate should return :const:`NotImplemented` from + :meth:`__add__`. (Or ``A`` may not implement :meth:`__add__` at + all.) +3. Then ``B``'s :meth:`__radd__` gets a chance. If it accepts + ``a``, all is well. +4. If it falls back to the boilerplate, there are no more possible + methods to try, so this is where the default implementation + should live. +5. If ``B <: A``, Python tries ``B.__radd__`` before + ``A.__add__``. This is ok, because it was implemented with + knowledge of ``A``, so it can handle those instances before + delegating to :class:`Complex`. If ``A <: Complex`` and ``B <: Real`` without sharing any other knowledge, then the appropriate shared operation is the one involving the built diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 6d90473f2367b5..96f2c287875d41 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -59,9 +59,9 @@ truth tests, identity tests, and boolean operations: __not__(obj) Return the outcome of :keyword:`not` *obj*. (Note that there is no - :meth:`__not__` method for object instances; only the interpreter core defines - this operation. The result is affected by the :meth:`__bool__` and - :meth:`__len__` methods.) + :meth:`!__not__` method for object instances; only the interpreter core defines + this operation. The result is affected by the :meth:`~object.__bool__` and + :meth:`~object.__len__` methods.) .. function:: truth(obj) @@ -244,7 +244,7 @@ Operations which work with sequences (some of them with mappings too) include: .. function:: length_hint(obj, default=0) - Return an estimated length for the object *o*. First try to return its + Return an estimated length for the object *obj*. First try to return its actual length, then an estimate using :meth:`object.__length_hint__`, and finally return the default value. @@ -306,7 +306,7 @@ expect a function argument. itemgetter(*items) Return a callable object that fetches *item* from its operand using the - operand's :meth:`__getitem__` method. If multiple items are specified, + operand's :meth:`~object.__getitem__` method. If multiple items are specified, returns a tuple of lookup values. For example: * After ``f = itemgetter(2)``, the call ``f(r)`` returns ``r[2]``. @@ -326,7 +326,7 @@ expect a function argument. return tuple(obj[item] for item in items) return g - The items can be any type accepted by the operand's :meth:`__getitem__` + The items can be any type accepted by the operand's :meth:`~object.__getitem__` method. Dictionaries accept any :term:`hashable` value. Lists, tuples, and strings accept an index or a slice: diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 5c02d8bc8835bf..015e83ed2ce5f7 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -11,8 +11,9 @@ **Source code:** :source:`Lib/optparse.py` .. deprecated:: 3.2 - The :mod:`optparse` module is deprecated and will not be developed further; - development will continue with the :mod:`argparse` module. + The :mod:`optparse` module is :term:`soft deprecated` and will not be + developed further; development will continue with the :mod:`argparse` + module. -------------- @@ -42,8 +43,8 @@ on the command-line, for example:: --file=outfile -q As it parses the command line, :mod:`optparse` sets attributes of the -``options`` object returned by :meth:`parse_args` based on user-supplied -command-line values. When :meth:`parse_args` returns from parsing this command +``options`` object returned by :meth:`~OptionParser.parse_args` based on user-supplied +command-line values. When :meth:`~OptionParser.parse_args` returns from parsing this command line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be ``False``. :mod:`optparse` supports both long and short options, allows short options to be merged together, and allows options to be associated with their @@ -285,10 +286,10 @@ program's command line:: (options, args) = parser.parse_args() -(If you like, you can pass a custom argument list to :meth:`parse_args`, but +(If you like, you can pass a custom argument list to :meth:`~OptionParser.parse_args`, but that's rarely necessary: by default it uses ``sys.argv[1:]``.) -:meth:`parse_args` returns two values: +:meth:`~OptionParser.parse_args` returns two values: * ``options``, an object containing values for all of your options---e.g. if ``--file`` takes a single string argument, then ``options.file`` will be the @@ -339,7 +340,7 @@ Now let's make up a fake command line and ask :mod:`optparse` to parse it:: When :mod:`optparse` sees the option string ``-f``, it consumes the next argument, ``foo.txt``, and stores it in ``options.filename``. So, after this -call to :meth:`parse_args`, ``options.filename`` is ``"foo.txt"``. +call to :meth:`~OptionParser.parse_args`, ``options.filename`` is ``"foo.txt"``. Some other option types supported by :mod:`optparse` are ``int`` and ``float``. Here's an option that expects an integer argument:: @@ -453,7 +454,8 @@ Again, the default value for ``verbose`` will be ``True``: the last default value supplied for any particular destination is the one that counts. A clearer way to specify default values is the :meth:`set_defaults` method of -OptionParser, which you can call at any time before calling :meth:`parse_args`:: +OptionParser, which you can call at any time before calling +:meth:`~OptionParser.parse_args`:: parser.set_defaults(verbose=True) parser.add_option(...) @@ -811,7 +813,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. help option. When :mod:`optparse` prints the usage string, it expands ``%prog`` to ``os.path.basename(sys.argv[0])`` (or to ``prog`` if you passed that keyword argument). To suppress a usage message, pass the - special value :data:`optparse.SUPPRESS_USAGE`. + special value :const:`optparse.SUPPRESS_USAGE`. ``option_list`` (default: ``[]``) A list of Option objects to populate the parser with. The options in @@ -1077,7 +1079,7 @@ relevant to a particular option, or fail to pass a required option attribute, Help text to print for this option when listing all available options after the user supplies a :attr:`~Option.help` option (such as ``--help``). If no help text is supplied, the option will be listed without help text. To - hide this option, use the special value :data:`optparse.SUPPRESS_HELP`. + hide this option, use the special value :const:`optparse.SUPPRESS_HELP`. .. attribute:: Option.metavar @@ -1249,7 +1251,7 @@ must specify for any option using that action. If no :attr:`~Option.help` string is supplied for an option, it will still be listed in the help message. To omit an option entirely, use the special value - :data:`optparse.SUPPRESS_HELP`. + :const:`optparse.SUPPRESS_HELP`. :mod:`optparse` automatically adds a :attr:`~Option.help` option to all OptionParsers, so you do not normally need to create one. @@ -1338,35 +1340,37 @@ Parsing arguments ^^^^^^^^^^^^^^^^^ The whole point of creating and populating an OptionParser is to call its -:meth:`parse_args` method:: +:meth:`~OptionParser.parse_args` method. - (options, args) = parser.parse_args(args=None, values=None) +.. method:: OptionParser.parse_args(args=None, values=None) -where the input parameters are + Parse the command-line options found in *args*. -``args`` - the list of arguments to process (default: ``sys.argv[1:]``) + The input parameters are -``values`` - an :class:`optparse.Values` object to store option arguments in (default: a - new instance of :class:`Values`) -- if you give an existing object, the - option defaults will not be initialized on it + ``args`` + the list of arguments to process (default: ``sys.argv[1:]``) -and the return values are + ``values`` + an :class:`Values` object to store option arguments in (default: a + new instance of :class:`Values`) -- if you give an existing object, the + option defaults will not be initialized on it -``options`` - the same object that was passed in as ``values``, or the optparse.Values - instance created by :mod:`optparse` + and the return value is a pair ``(options, args)`` where -``args`` - the leftover positional arguments after all options have been processed + ``options`` + the same object that was passed in as *values*, or the ``optparse.Values`` + instance created by :mod:`optparse` + + ``args`` + the leftover positional arguments after all options have been processed The most common usage is to supply neither keyword argument. If you supply ``values``, it will be modified with repeated :func:`setattr` calls (roughly one for every option argument stored to an option destination) and returned by -:meth:`parse_args`. +:meth:`~OptionParser.parse_args`. -If :meth:`parse_args` encounters any errors in the argument list, it calls the +If :meth:`~OptionParser.parse_args` encounters any errors in the argument list, it calls the OptionParser's :meth:`error` method with an appropriate end-user error message. This ultimately terminates your process with an exit status of 2 (the traditional Unix exit status for command-line errors). @@ -1518,7 +1522,7 @@ OptionParser supports several other public methods: Set the usage string according to the rules described above for the ``usage`` constructor keyword argument. Passing ``None`` sets the default usage - string; use :data:`optparse.SUPPRESS_USAGE` to suppress a usage message. + string; use :const:`optparse.SUPPRESS_USAGE` to suppress a usage message. .. method:: OptionParser.print_usage(file=None) @@ -1661,7 +1665,7 @@ where the current list of leftover arguments, ie. arguments that have been consumed but are neither options nor option arguments. Feel free to modify ``parser.largs``, e.g. by adding more arguments to it. (This list will - become ``args``, the second return value of :meth:`parse_args`.) + become ``args``, the second return value of :meth:`~OptionParser.parse_args`.) ``parser.rargs`` the current list of remaining arguments, ie. with ``opt_str`` and diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 7881c52db87090..95933f56d50542 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -304,6 +304,24 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. +.. function:: isdevdrive(path) + + Return ``True`` if pathname *path* is located on a Windows Dev Drive. + A Dev Drive is optimized for developer scenarios, and offers faster + performance for reading and writing files. It is recommended for use for + source code, temporary build directories, package caches, and other + IO-intensive operations. + + May raise an error for an invalid path, for example, one without a + recognizable drive, but returns ``False`` on platforms that do not support + Dev Drives. See `the Windows documentation `_ + for information on enabling and creating Dev Drives. + + .. availability:: Windows. + + .. versionadded:: 3.12 + + .. function:: join(path, *paths) Join one or more path segments intelligently. The return value is the @@ -359,7 +377,8 @@ the :mod:`glob` module.) Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path (if they are supported by the operating - system). + system). On Windows, this function will also resolve MS-DOS (also called 8.3) + style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. If a path doesn't exist or a symlink loop is encountered, and *strict* is ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is @@ -392,7 +411,7 @@ the :mod:`glob` module.) *start*. On Windows, :exc:`ValueError` is raised when *path* and *start* are on different drives. - *start* defaults to :attr:`os.curdir`. + *start* defaults to :data:`os.curdir`. .. availability:: Unix, Windows. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 641e289e77c518..fe573f188ab066 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -60,7 +60,7 @@ Notes on the availability of these functions: ``'java'``. .. seealso:: - :attr:`sys.platform` has a finer granularity. :func:`os.uname` gives + :data:`sys.platform` has a finer granularity. :func:`os.uname` gives system-dependent version information. The :mod:`platform` module provides detailed checks for the @@ -88,8 +88,8 @@ startup by the :c:func:`PyConfig_Read` function: see On some systems, conversion using the file system encoding may fail. In this case, Python uses the :ref:`surrogateescape encoding error handler `, which means that undecodable bytes are replaced by a - Unicode character U+DCxx on decoding, and these are again translated to the - original byte on encoding. + Unicode character U+DC\ *xx* on decoding, and these are again + translated to the original byte on encoding. The :term:`file system encoding ` must @@ -215,7 +215,7 @@ process and user. On some platforms, including FreeBSD and macOS, setting ``environ`` may cause memory leaks. Refer to the system documentation for - :c:func:`putenv`. + :c:func:`!putenv`. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from @@ -233,7 +233,7 @@ process and user. :data:`environ` and :data:`environb` are synchronized (modifying :data:`environb` updates :data:`environ`, and vice versa). - :data:`environb` is only available if :data:`supports_bytes_environ` is + :data:`environb` is only available if :const:`supports_bytes_environ` is ``True``. .. versionadded:: 3.2 @@ -331,7 +331,7 @@ process and user. future environment changes. - :func:`getenvb` is only available if :data:`supports_bytes_environ` + :func:`getenvb` is only available if :const:`supports_bytes_environ` is ``True``. .. availability:: Unix. @@ -401,11 +401,11 @@ process and user. On macOS, :func:`getgroups` behavior differs somewhat from other Unix platforms. If the Python interpreter was built with a - deployment target of :const:`10.5` or earlier, :func:`getgroups` returns + deployment target of ``10.5`` or earlier, :func:`getgroups` returns the list of effective group ids associated with the current user process; this list is limited to a system-defined number of entries, typically 16, and may be modified by calls to :func:`setgroups` if suitably privileged. - If built with a deployment target greater than :const:`10.5`, + If built with a deployment target greater than ``10.5``, :func:`getgroups` returns the current group access list for the user associated with the effective user id of the process; the group access list may change over the lifetime of the process, it is not affected by @@ -493,6 +493,17 @@ process and user. .. versionadded:: 3.3 +.. data:: PRIO_DARWIN_THREAD + PRIO_DARWIN_PROCESS + PRIO_DARWIN_BG + PRIO_DARWIN_NONUI + + Parameters for the :func:`getpriority` and :func:`setpriority` functions. + + .. availability:: macOS + + .. versionadded:: 3.12 + .. function:: getresuid() Return a tuple (ruid, euid, suid) denoting the current process's @@ -553,7 +564,7 @@ process and user. .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may - cause memory leaks. Refer to the system documentation for :c:func:`putenv`. + cause memory leaks. Refer to the system documentation for :c:func:`!putenv`. .. audit-event:: os.putenv key,value os.putenv @@ -635,7 +646,7 @@ process and user. .. function:: setpgrp() - Call the system call :c:func:`setpgrp` or ``setpgrp(0, 0)`` depending on + Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on which version is implemented (if any). See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. @@ -643,7 +654,7 @@ process and user. .. function:: setpgid(pid, pgrp, /) - Call the system call :c:func:`setpgid` to set the process group id of the + Call the system call :c:func:`!setpgid` to set the process group id of the process with id *pid* to the process group with id *pgrp*. See the Unix manual for the semantics. @@ -703,14 +714,14 @@ process and user. .. function:: getsid(pid, /) - Call the system call :c:func:`getsid`. See the Unix manual for the semantics. + Call the system call :c:func:`!getsid`. See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. .. function:: setsid() - Call the system call :c:func:`setsid`. See the Unix manual for the semantics. + Call the system call :c:func:`!setsid`. See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. @@ -728,7 +739,7 @@ process and user. .. function:: strerror(code, /) Return the error message corresponding to the error code in *code*. - On platforms where :c:func:`strerror` returns ``NULL`` when given an unknown + On platforms where :c:func:`!strerror` returns ``NULL`` when given an unknown error number, :exc:`ValueError` is raised. @@ -912,7 +923,7 @@ as internal buffering of data. In Linux kernel older than 5.3, the files pointed by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is - raised with :attr:`~OSError.errno` set to :data:`errno.EXDEV`. + raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. This copy is done without the additional cost of transferring data from the kernel to user space and then back into the kernel. Additionally, @@ -1066,7 +1077,7 @@ as internal buffering of data. .. function:: fsync(fd) Force write of file with filedescriptor *fd* to disk. On Unix, this calls the - native :c:func:`fsync` function; on Windows, the MS :c:func:`_commit` function. + native :c:func:`!fsync` function; on Windows, the MS :c:func:`!_commit` function. If you're starting with a buffered Python :term:`file object` *f*, first do ``f.flush()``, and then do ``os.fsync(f.fileno())``, to ensure that all internal @@ -1152,25 +1163,65 @@ as internal buffering of data. .. versionadded:: 3.11 -.. function:: lseek(fd, pos, how, /) +.. function:: lseek(fd, pos, whence, /) Set the current position of file descriptor *fd* to position *pos*, modified - by *how*: :const:`SEEK_SET` or ``0`` to set the position relative to the - beginning of the file; :const:`SEEK_CUR` or ``1`` to set it relative to the - current position; :const:`SEEK_END` or ``2`` to set it relative to the end of - the file. Return the new cursor position in bytes, starting from the beginning. + by *whence*, and return the new position in bytes relative to + the start of the file. + Valid values for *whence* are: + + * :const:`SEEK_SET` or ``0`` -- set *pos* relative to the beginning of the file + * :const:`SEEK_CUR` or ``1`` -- set *pos* relative to the current file position + * :const:`SEEK_END` or ``2`` -- set *pos* relative to the end of the file + * :const:`SEEK_HOLE` -- set *pos* to the next data location, relative to *pos* + * :const:`SEEK_DATA` -- set *pos* to the next data hole, relative to *pos* + + .. versionchanged:: 3.3 + + Add support for :const:`!SEEK_HOLE` and :const:`!SEEK_DATA`. .. data:: SEEK_SET SEEK_CUR SEEK_END - Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, - respectively. + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on :term:`file-like objects `, + for whence to adjust the file position indicator. + + :const:`SEEK_SET` + Adjust the file position relative to the beginning of the file. + :const:`SEEK_CUR` + Adjust the file position relative to the current file position. + :const:`SEEK_END` + Adjust the file position relative to the end of the file. + + Their values are 0, 1, and 2, respectively. + + +.. data:: SEEK_HOLE + SEEK_DATA + + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on :term:`file-like objects `, + for seeking file data and holes on sparsely allocated files. + + :data:`!SEEK_DATA` + Adjust the file offset to the next location containing data, + relative to the seek position. + + :data:`!SEEK_HOLE` + Adjust the file offset to the next location containing a hole, + relative to the seek position. + A hole is defined as a sequence of zeros. + + .. note:: + + These operations only make sense for filesystems that support them. + + .. availability:: Linux >= 3.1, macOS, Unix .. versionadded:: 3.3 - Some operating systems could support additional values, like - :data:`os.SEEK_HOLE` or :data:`os.SEEK_DATA`. .. function:: open(path, flags, mode=0o777, *, dir_fd=None) @@ -1411,7 +1462,7 @@ or `the MSDN `_ on Windo If some data was successfully read, it will return the number of bytes read. If no bytes were read, it will return ``-1`` and set errno to - :data:`errno.EAGAIN`. + :const:`errno.EAGAIN`. .. availability:: Linux >= 4.14. @@ -1567,25 +1618,6 @@ or `the MSDN `_ on Windo Parameters *out* and *in* was renamed to *out_fd* and *in_fd*. -.. function:: set_blocking(fd, blocking, /) - - Set the blocking mode of the specified file descriptor. Set the - :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. - - See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - - .. availability:: Unix, Windows. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - On Windows, this function is limited to pipes. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.12 - Added support for pipes on Windows. - .. data:: SF_NODISKIO SF_MNOWAIT SF_SYNC @@ -1607,6 +1639,26 @@ or `the MSDN `_ on Windo .. versionadded:: 3.11 +.. function:: set_blocking(fd, blocking, /) + + Set the blocking mode of the specified file descriptor. Set the + :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. + + See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. + + .. availability:: Unix, Windows. + + The function is limited on Emscripten and WASI, see + :ref:`wasm-availability` for more information. + + On Windows, this function is limited to pipes. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.12 + Added support for pipes on Windows. + + .. function:: splice(src, dst, count, offset_src=None, offset_dst=None) Transfer *count* bytes from file descriptor *src*, starting from offset @@ -1616,7 +1668,7 @@ or `the MSDN `_ on Windo *offset_dst*. The offset associated to the file descriptor that refers to a pipe must be ``None``. The files pointed by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with - :attr:`~OSError.errno` set to :data:`errno.EXDEV`. + :attr:`~OSError.errno` set to :const:`errno.EXDEV`. This copy is done without the additional cost of transferring data from the kernel to user space and then back into the kernel. Additionally, @@ -1949,18 +2001,18 @@ features: Set the flags of *path* to the numeric *flags*. *flags* may take a combination (bitwise OR) of the following values (as defined in the :mod:`stat` module): - * :data:`stat.UF_NODUMP` - * :data:`stat.UF_IMMUTABLE` - * :data:`stat.UF_APPEND` - * :data:`stat.UF_OPAQUE` - * :data:`stat.UF_NOUNLINK` - * :data:`stat.UF_COMPRESSED` - * :data:`stat.UF_HIDDEN` - * :data:`stat.SF_ARCHIVED` - * :data:`stat.SF_IMMUTABLE` - * :data:`stat.SF_APPEND` - * :data:`stat.SF_NOUNLINK` - * :data:`stat.SF_SNAPSHOT` + * :const:`stat.UF_NODUMP` + * :const:`stat.UF_IMMUTABLE` + * :const:`stat.UF_APPEND` + * :const:`stat.UF_OPAQUE` + * :const:`stat.UF_NOUNLINK` + * :const:`stat.UF_COMPRESSED` + * :const:`stat.UF_HIDDEN` + * :const:`stat.SF_ARCHIVED` + * :const:`stat.SF_IMMUTABLE` + * :const:`stat.SF_APPEND` + * :const:`stat.SF_NOUNLINK` + * :const:`stat.SF_SNAPSHOT` This function can support :ref:`not following symlinks `. @@ -1981,25 +2033,25 @@ features: following values (as defined in the :mod:`stat` module) or bitwise ORed combinations of them: - * :data:`stat.S_ISUID` - * :data:`stat.S_ISGID` - * :data:`stat.S_ENFMT` - * :data:`stat.S_ISVTX` - * :data:`stat.S_IREAD` - * :data:`stat.S_IWRITE` - * :data:`stat.S_IEXEC` - * :data:`stat.S_IRWXU` - * :data:`stat.S_IRUSR` - * :data:`stat.S_IWUSR` - * :data:`stat.S_IXUSR` - * :data:`stat.S_IRWXG` - * :data:`stat.S_IRGRP` - * :data:`stat.S_IWGRP` - * :data:`stat.S_IXGRP` - * :data:`stat.S_IRWXO` - * :data:`stat.S_IROTH` - * :data:`stat.S_IWOTH` - * :data:`stat.S_IXOTH` + * :const:`stat.S_ISUID` + * :const:`stat.S_ISGID` + * :const:`stat.S_ENFMT` + * :const:`stat.S_ISVTX` + * :const:`stat.S_IREAD` + * :const:`stat.S_IWRITE` + * :const:`stat.S_IEXEC` + * :const:`stat.S_IRWXU` + * :const:`stat.S_IRUSR` + * :const:`stat.S_IWUSR` + * :const:`stat.S_IXUSR` + * :const:`stat.S_IRWXG` + * :const:`stat.S_IRGRP` + * :const:`stat.S_IWGRP` + * :const:`stat.S_IXGRP` + * :const:`stat.S_IRWXO` + * :const:`stat.S_IROTH` + * :const:`stat.S_IWOTH` + * :const:`stat.S_IXOTH` This function can support :ref:`specifying a file descriptor `, :ref:`paths relative to directory descriptors ` and :ref:`not @@ -2139,7 +2191,7 @@ features: .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link - .. availability:: Unix, Windows. + .. availability:: Unix, Windows, not Emscripten. .. versionchanged:: 3.2 Added Windows support. @@ -2253,7 +2305,7 @@ features: .. function:: lstat(path, *, dir_fd=None) - Perform the equivalent of an :c:func:`lstat` system call on the given path. + Perform the equivalent of an :c:func:`!lstat` system call on the given path. Similar to :func:`~os.stat`, but does not follow symbolic links. Return a :class:`stat_result` object. @@ -2409,13 +2461,13 @@ features: .. function:: major(device, /) Extract the device major number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:type:`stat`). + :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). .. function:: minor(device, /) Extract the device minor number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:type:`stat`). + :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). .. function:: makedev(major, minor, /) @@ -2926,7 +2978,7 @@ features: .. class:: stat_result Object whose attributes correspond roughly to the members of the - :c:type:`stat` structure. It is used for the result of :func:`os.stat`, + :c:struct:`stat` structure. It is used for the result of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat`. Attributes: @@ -3096,22 +3148,24 @@ features: Windows file attributes: ``dwFileAttributes`` member of the ``BY_HANDLE_FILE_INFORMATION`` structure returned by - :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` + :c:func:`!GetFileInformationByHandle`. + See the :const:`!FILE_ATTRIBUTE_* ` constants in the :mod:`stat` module. .. attribute:: st_reparse_tag - When :attr:`st_file_attributes` has the ``FILE_ATTRIBUTE_REPARSE_POINT`` + When :attr:`st_file_attributes` has the :const:`~stat.FILE_ATTRIBUTE_REPARSE_POINT` set, this field contains the tag identifying the type of reparse point. - See the ``IO_REPARSE_TAG_*`` constants in the :mod:`stat` module. + See the :const:`IO_REPARSE_TAG_* ` + constants in the :mod:`stat` module. The standard module :mod:`stat` defines functions and constants that are - useful for extracting information from a :c:type:`stat` structure. (On + useful for extracting information from a :c:struct:`stat` structure. (On Windows, some items are filled with dummy values.) For backward compatibility, a :class:`stat_result` instance is also accessible as a tuple of at least 10 integers giving the most important (and - portable) members of the :c:type:`stat` structure, in the order + portable) members of the :c:struct:`stat` structure, in the order :attr:`st_mode`, :attr:`st_ino`, :attr:`st_dev`, :attr:`st_nlink`, :attr:`st_uid`, :attr:`st_gid`, :attr:`st_size`, :attr:`st_atime`, :attr:`st_mtime`, :attr:`st_ctime`. More items may be added at the end by @@ -3161,9 +3215,9 @@ features: .. function:: statvfs(path) - Perform a :c:func:`statvfs` system call on the given path. The return value is + Perform a :c:func:`!statvfs` system call on the given path. The return value is an object whose attributes describe the filesystem on the given path, and - correspond to the members of the :c:type:`statvfs` structure, namely: + correspond to the members of the :c:struct:`statvfs` structure, namely: :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, :attr:`f_flag`, :attr:`f_namemax`, :attr:`f_fsid`. @@ -3727,6 +3781,217 @@ features: .. versionadded:: 3.10 +Timer File Descriptors +~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 3.13 + +These functions provide support for Linux's *timer file descriptor* API. +Naturally, they are all only available on Linux. + +.. function:: timerfd_create(clockid, /, *, flags=0) + + Create and return a timer file descriptor (*timerfd*). + + The file descriptor returned by :func:`timerfd_create` supports: + + - :func:`read` + - :func:`~select.select` + - :func:`~select.poll` + + The file descriptor's :func:`read` method can be called with a buffer size + of 8. If the timer has already expired one or more times, :func:`read` + returns the number of expirations with the host's endianness, which may be + converted to an :class:`int` by ``int.from_bytes(x, byteorder=sys.byteorder)``. + + :func:`~select.select` and :func:`~select.poll` can be used to wait until + timer expires and the file descriptor is readable. + + *clockid* must be a valid :ref:`clock ID `, + as defined in the :py:mod:`time` module: + + - :const:`time.CLOCK_REALTIME` + - :const:`time.CLOCK_MONOTONIC` + - :const:`time.CLOCK_BOOTTIME` (Since Linux 3.15 for timerfd_create) + + If *clockid* is :const:`time.CLOCK_REALTIME`, a settable system-wide + real-time clock is used. If system clock is changed, timer setting need + to be updated. To cancel timer when system clock is changed, see + :const:`TFD_TIMER_CANCEL_ON_SET`. + + If *clockid* is :const:`time.CLOCK_MONOTONIC`, a non-settable monotonically + increasing clock is used. Even if the system clock is changed, the timer + setting will not be affected. + + If *clockid* is :const:`time.CLOCK_BOOTTIME`, same as :const:`time.CLOCK_MONOTONIC` + except it includes any time that the system is suspended. + + The file descriptor's behaviour can be modified by specifying a *flags* value. + Any of the following variables may used, combined using bitwise OR + (the ``|`` operator): + + - :const:`TFD_NONBLOCK` + - :const:`TFD_CLOEXEC` + + If :const:`TFD_NONBLOCK` is not set as a flag, :func:`read` blocks until + the timer expires. If it is set as a flag, :func:`read` doesn't block, but + If there hasn't been an expiration since the last call to read, + :func:`read` raises :class:`OSError` with ``errno`` is set to + :const:`errno.EAGAIN`. + + :const:`TFD_CLOEXEC` is always set by Python automatically. + + The file descriptor must be closed with :func:`os.close` when it is no + longer needed, or else the file descriptor will be leaked. + + .. seealso:: The :manpage:`timerfd_create(2)` man page. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + +.. function:: timerfd_settime(fd, /, *, flags=flags, initial=0.0, interval=0.0) + + Alter a timer file descriptor's internal timer. + This function operates the same interval timer as :func:`timerfd_settime_ns`. + + *fd* must be a valid timer file descriptor. + + The timer's behaviour can be modified by specifying a *flags* value. + Any of the following variables may used, combined using bitwise OR + (the ``|`` operator): + + - :const:`TFD_TIMER_ABSTIME` + - :const:`TFD_TIMER_CANCEL_ON_SET` + + The timer is disabled by setting *initial* to zero (``0``). + If *initial* is equal to or greater than zero, the timer is enabled. + If *initial* is less than zero, it raises an :class:`OSError` exception + with ``errno`` set to :const:`errno.EINVAL` + + By default the timer will fire when *initial* seconds have elapsed. + (If *initial* is zero, timer will fire immediately.) + + However, if the :const:`TFD_TIMER_ABSTIME` flag is set, + the timer will fire when the timer's clock + (set by *clockid* in :func:`timerfd_create`) reaches *initial* seconds. + + The timer's interval is set by the *interval* :py:class:`float`. + If *interval* is zero, the timer only fires once, on the initial expiration. + If *interval* is greater than zero, the timer fires every time *interval* + seconds have elapsed since the previous expiration. + If *interval* is less than zero, it raises :class:`OSError` with ``errno`` + set to :const:`errno.EINVAL` + + If the :const:`TFD_TIMER_CANCEL_ON_SET` flag is set along with + :const:`TFD_TIMER_ABSTIME` and the clock for this timer is + :const:`time.CLOCK_REALTIME`, the timer is marked as cancelable if the + real-time clock is changed discontinuously. Reading the descriptor is + aborted with the error ECANCELED. + + Linux manages system clock as UTC. A daylight-savings time transition is + done by changing time offset only and doesn't cause discontinuous system + clock change. + + Discontinuous system clock change will be caused by the following events: + + - ``settimeofday`` + - ``clock_settime`` + - set the system date and time by ``date`` command + + Return a two-item tuple of (``next_expiration``, ``interval``) from + the previous timer state, before this function executed. + + .. seealso:: + + :manpage:`timerfd_create(2)`, :manpage:`timerfd_settime(2)`, + :manpage:`settimeofday(2)`, :manpage:`clock_settime(2)`, + and :manpage:`date(1)`. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + +.. function:: timerfd_settime_ns(fd, /, *, flags=0, initial=0, interval=0) + + Similar to :func:`timerfd_settime`, but use time as nanoseconds. + This function operates the same interval timer as :func:`timerfd_settime`. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + +.. function:: timerfd_gettime(fd, /) + + Return a two-item tuple of floats (``next_expiration``, ``interval``). + + ``next_expiration`` denotes the relative time until next the timer next fires, + regardless of if the :const:`TFD_TIMER_ABSTIME` flag is set. + + ``interval`` denotes the timer's interval. + If zero, the timer will only fire once, after ``next_expiration`` seconds + have elapsed. + + .. seealso:: :manpage:`timerfd_gettime(2)` + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + +.. function:: timerfd_gettime_ns(fd, /) + + Similar to :func:`timerfd_gettime`, but return time as nanoseconds. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + +.. data:: TFD_NONBLOCK + + A flag for the :func:`timerfd_create` function, + which sets the :const:`O_NONBLOCK` status flag for the new timer file + descriptor. If :const:`TFD_NONBLOCK` is not set as a flag, :func:`read` blocks. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + +.. data:: TFD_CLOEXEC + + A flag for the :func:`timerfd_create` function, + If :const:`TFD_CLOEXEC` is set as a flag, set close-on-exec flag for new file + descriptor. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + +.. data:: TFD_TIMER_ABSTIME + + A flag for the :func:`timerfd_settime` and :func:`timerfd_settime_ns` functions. + If this flag is set, *initial* is interpreted as an absolute value on the + timer's clock (in UTC seconds or nanoseconds since the Unix Epoch). + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + +.. data:: TFD_TIMER_CANCEL_ON_SET + + A flag for the :func:`timerfd_settime` and :func:`timerfd_settime_ns` + functions along with :const:`TFD_TIMER_ABSTIME`. + The timer is cancelled when the time of the underlying clock changes + discontinuously. + + .. availability:: Linux >= 2.6.27 with glibc >= 2.8 + + .. versionadded:: 3.13 + + Linux extended attributes ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -4103,15 +4368,38 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.fork "" os.fork + .. warning:: + + If you use TLS sockets in an application calling ``fork()``, see + the warning in the :mod:`ssl` documentation. + .. versionchanged:: 3.8 Calling ``fork()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). - .. warning:: - - See :mod:`ssl` for applications that use the SSL module with fork(). + .. versionchanged:: 3.12 + If Python is able to detect that your process has multiple + threads, :func:`os.fork` now raises a :exc:`DeprecationWarning`. + + We chose to surface this as a warning, when detectable, to better + inform developers of a design problem that the POSIX platform + specifically notes as not supported. Even in code that + *appears* to work, it has never been safe to mix threading with + :func:`os.fork` on POSIX platforms. The CPython runtime itself has + always made API calls that are not safe for use in the child + process when threads existed in the parent (such as ``malloc`` and + ``free``). + + Users of macOS or users of libc or malloc implementations other + than those typically found in glibc to date are among those + already more likely to experience deadlocks running such code. + + See `this discussion on fork being incompatible with threads + `_ + for technical details of why we're surfacing this longstanding + platform compatibility problem to developers. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: POSIX, not Emscripten, not WASI. .. function:: forkpty() @@ -4124,6 +4412,11 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.forkpty "" os.forkpty + .. versionchanged:: 3.12 + If Python is able to detect that your process has multiple + threads, this now raises a :exc:`DeprecationWarning`. See the + longer explanation on :func:`os.fork`. + .. versionchanged:: 3.8 Calling ``forkpty()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). @@ -4140,8 +4433,8 @@ written in Python, such as a mail server's external command delivery program. Send signal *sig* to the process *pid*. Constants for the specific signals available on the host platform are defined in the :mod:`signal` module. - Windows: The :data:`signal.CTRL_C_EVENT` and - :data:`signal.CTRL_BREAK_EVENT` signals are special signals which can + Windows: The :const:`signal.CTRL_C_EVENT` and + :const:`signal.CTRL_BREAK_EVENT` signals are special signals which can only be sent to console processes which share a common console window, e.g., some subprocesses. Any other value for *sig* will cause the process to be unconditionally killed by the TerminateProcess API, and the exit code @@ -4194,7 +4487,7 @@ written in Python, such as a mail server's external command delivery program. This flag indicates that the file descriptor will be non-blocking. If the process referred to by the file descriptor has not yet terminated, then an attempt to wait on the file descriptor using :manpage:`waitid(2)` - will immediately return the error :data:`~errno.EAGAIN` rather than blocking. + will immediately return the error :const:`~errno.EAGAIN` rather than blocking. .. availability:: Linux >= 5.10 .. versionadded:: 3.12 @@ -4252,7 +4545,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawn` C library API for use from Python. + Wraps the :c:func:`!posix_spawn` C library API for use from Python. Most users should use :func:`subprocess.run` instead of :func:`posix_spawn`. @@ -4288,16 +4581,16 @@ written in Python, such as a mail server's external command delivery program. Performs ``os.dup2(fd, new_fd)``. These tuples correspond to the C library - :c:func:`posix_spawn_file_actions_addopen`, - :c:func:`posix_spawn_file_actions_addclose`, and - :c:func:`posix_spawn_file_actions_adddup2` API calls used to prepare - for the :c:func:`posix_spawn` call itself. + :c:func:`!posix_spawn_file_actions_addopen`, + :c:func:`!posix_spawn_file_actions_addclose`, and + :c:func:`!posix_spawn_file_actions_adddup2` API calls used to prepare + for the :c:func:`!posix_spawn` call itself. The *setpgroup* argument will set the process group of the child to the value specified. If the value specified is 0, the child's process group ID will be made the same as its process ID. If the value of *setpgroup* is not set, the child will inherit the parent's process group ID. This argument corresponds - to the C library :c:data:`POSIX_SPAWN_SETPGROUP` flag. + to the C library :c:macro:`!POSIX_SPAWN_SETPGROUP` flag. If the *resetids* argument is ``True`` it will reset the effective UID and GID of the child to the real UID and GID of the parent process. If the @@ -4305,27 +4598,27 @@ written in Python, such as a mail server's external command delivery program. the parent. In either case, if the set-user-ID and set-group-ID permission bits are enabled on the executable file, their effect will override the setting of the effective UID and GID. This argument corresponds to the C - library :c:data:`POSIX_SPAWN_RESETIDS` flag. + library :c:macro:`!POSIX_SPAWN_RESETIDS` flag. If the *setsid* argument is ``True``, it will create a new session ID - for ``posix_spawn``. *setsid* requires :c:data:`POSIX_SPAWN_SETSID` - or :c:data:`POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` + for ``posix_spawn``. *setsid* requires :c:macro:`!POSIX_SPAWN_SETSID` + or :c:macro:`!POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` is raised. The *setsigmask* argument will set the signal mask to the signal set specified. If the parameter is not used, then the child inherits the parent's signal mask. This argument corresponds to the C library - :c:data:`POSIX_SPAWN_SETSIGMASK` flag. + :c:macro:`!POSIX_SPAWN_SETSIGMASK` flag. The *sigdef* argument will reset the disposition of all signals in the set specified. This argument corresponds to the C library - :c:data:`POSIX_SPAWN_SETSIGDEF` flag. + :c:macro:`!POSIX_SPAWN_SETSIGDEF` flag. The *scheduler* argument must be a tuple containing the (optional) scheduler policy and an instance of :class:`sched_param` with the scheduler parameters. A value of ``None`` in the place of the scheduler policy indicates that is not being provided. This argument is a combination of the C library - :c:data:`POSIX_SPAWN_SETSCHEDPARAM` and :c:data:`POSIX_SPAWN_SETSCHEDULER` + :c:macro:`!POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`!POSIX_SPAWN_SETSCHEDULER` flags. .. audit-event:: os.posix_spawn path,argv,env os.posix_spawn @@ -4338,7 +4631,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawnp` C library API for use from Python. + Wraps the :c:func:`!posix_spawnp` C library API for use from Python. Similar to :func:`posix_spawn` except that the system searches for the *executable* file in the list of directories specified by the @@ -4519,7 +4812,7 @@ written in Python, such as a mail server's external command delivery program. Use *show_cmd* to override the default window style. Whether this has any effect will depend on the application being launched. Values are integers as - supported by the Win32 :c:func:`ShellExecute` function. + supported by the Win32 :c:func:`!ShellExecute` function. :func:`startfile` returns as soon as the associated application is launched. There is no option to wait for the application to close, and no way to retrieve @@ -4529,7 +4822,7 @@ written in Python, such as a mail server's external command delivery program. :func:`os.path.normpath` function to ensure that paths are properly encoded for Win32. - To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute` + To reduce interpreter startup overhead, the Win32 :c:func:`!ShellExecute` function is not resolved until this function is first called. If the function cannot be resolved, :exc:`NotImplementedError` will be raised. @@ -4639,11 +4932,11 @@ written in Python, such as a mail server's external command delivery program. :data:`WNOHANG` and :data:`WNOWAIT` are additional optional flags. The return value is an object representing the data contained in the - :c:type:`!siginfo_t` structure with the following attributes: + :c:type:`siginfo_t` structure with the following attributes: * :attr:`!si_pid` (process ID) * :attr:`!si_uid` (real user ID of the child) - * :attr:`!si_signo` (always :data:`~signal.SIGCHLD`) + * :attr:`!si_signo` (always :const:`~signal.SIGCHLD`) * :attr:`!si_status` (the exit status or signal number, depending on :attr:`!si_code`) * :attr:`!si_code` (see :data:`CLD_EXITED` for possible values) @@ -4881,7 +5174,7 @@ used to determine the disposition of a process. .. function:: WIFCONTINUED(status) Return ``True`` if a stopped child has been resumed by delivery of - :data:`~signal.SIGCONT` (if the process has been continued from a job + :const:`~signal.SIGCONT` (if the process has been continued from a job control stop), otherwise return ``False``. See :data:`WCONTINUED` option. @@ -5059,8 +5352,12 @@ operating system. .. function:: sched_getaffinity(pid, /) - Return the set of CPUs the process with PID *pid* (or the current process - if zero) is restricted to. + Return the set of CPUs the process with PID *pid* is restricted to. + + If *pid* is zero, return the set of CPUs the calling thread of the current + process is restricted to. + + See also the :func:`process_cpu_count` function. .. _os-path: @@ -5101,15 +5398,18 @@ Miscellaneous System Information .. function:: cpu_count() - Return the number of CPUs in the system. Returns ``None`` if undetermined. - - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` + Return the number of logical CPUs in the **system**. Returns ``None`` if + undetermined. + The :func:`process_cpu_count` function can be used to get the number of + logical CPUs usable by the calling thread of the **current process**. .. versionadded:: 3.4 + .. versionchanged:: 3.13 + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, + :func:`cpu_count` returns the overridden value *n*. + .. function:: getloadavg() @@ -5120,6 +5420,23 @@ Miscellaneous System Information .. availability:: Unix. +.. function:: process_cpu_count() + + Get the number of logical CPUs usable by the calling thread of the **current + process**. Returns ``None`` if undetermined. It can be less than + :func:`cpu_count` depending on the CPU affinity. + + The :func:`cpu_count` function can be used to get the number of logical CPUs + in the **system**. + + If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, + :func:`process_cpu_count` returns the overridden value *n*. + + See also the :func:`sched_getaffinity` functions. + + .. versionadded:: 3.13 + + .. function:: sysconf(name, /) Return integer-valued system configuration values. If the configuration value @@ -5253,7 +5570,7 @@ Random numbers ``/dev/urandom`` devices. The flags argument is a bit mask that can contain zero or more of the - following values ORed together: :py:data:`os.GRND_RANDOM` and + following values ORed together: :py:const:`os.GRND_RANDOM` and :py:data:`GRND_NONBLOCK`. See also the `Linux getrandom() manual page diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst deleted file mode 100644 index e14c1bf8d5367e..00000000000000 --- a/Doc/library/ossaudiodev.rst +++ /dev/null @@ -1,453 +0,0 @@ -:mod:`ossaudiodev` --- Access to OSS-compatible audio devices -============================================================= - -.. module:: ossaudiodev - :platform: Linux, FreeBSD - :synopsis: Access to OSS-compatible audio devices. - :deprecated: - -.. deprecated-removed:: 3.11 3.13 - The :mod:`ossaudiodev` module is deprecated - (see :pep:`PEP 594 <594#ossaudiodev>` for details). - --------------- - -This module allows you to access the OSS (Open Sound System) audio interface. -OSS is available for a wide range of open-source and commercial Unices, and is -the standard audio interface for Linux and recent versions of FreeBSD. - -.. Things will get more complicated for future Linux versions, since - ALSA is in the standard kernel as of 2.5.x. Presumably if you - use ALSA, you'll have to make sure its OSS compatibility layer - is active to use ossaudiodev, but you're going to need it for the vast - majority of Linux audio apps anyway. - - Sounds like things are also complicated for other BSDs. In response - to my python-dev query, Thomas Wouters said: - - > Likewise, googling shows OpenBSD also uses OSS/Free -- the commercial - > OSS installation manual tells you to remove references to OSS/Free from the - > kernel :) - - but Aleksander Piotrowsk actually has an OpenBSD box, and he quotes - from its : - > * WARNING! WARNING! - > * This is an OSS (Linux) audio emulator. - > * Use the Native NetBSD API for developing new code, and this - > * only for compiling Linux programs. - - There's also an ossaudio manpage on OpenBSD that explains things - further. Presumably NetBSD and OpenBSD have a different standard - audio interface. That's the great thing about standards, there are so - many to choose from ... ;-) - - This probably all warrants a footnote or two, but I don't understand - things well enough right now to write it! --GPW - -.. versionchanged:: 3.3 - Operations in this module now raise :exc:`OSError` where :exc:`IOError` - was raised. - - -.. seealso:: - - `Open Sound System Programmer's Guide `_ - the official documentation for the OSS C API - - The module defines a large number of constants supplied by the OSS device - driver; see ```` on either Linux or FreeBSD for a listing. - -:mod:`ossaudiodev` defines the following variables and functions: - - -.. exception:: OSSAudioError - - This exception is raised on certain errors. The argument is a string describing - what went wrong. - - (If :mod:`ossaudiodev` receives an error from a system call such as - :c:func:`open`, :c:func:`write`, or :c:func:`ioctl`, it raises :exc:`OSError`. - Errors detected directly by :mod:`ossaudiodev` result in :exc:`OSSAudioError`.) - - (For backwards compatibility, the exception class is also available as - ``ossaudiodev.error``.) - - -.. function:: open(mode) - open(device, mode) - - Open an audio device and return an OSS audio device object. This object - supports many file-like methods, such as :meth:`read`, :meth:`write`, and - :meth:`fileno` (although there are subtle differences between conventional Unix - read/write semantics and those of OSS audio devices). It also supports a number - of audio-specific methods; see below for the complete list of methods. - - *device* is the audio device filename to use. If it is not specified, this - module first looks in the environment variable :envvar:`AUDIODEV` for a device - to use. If not found, it falls back to :file:`/dev/dsp`. - - *mode* is one of ``'r'`` for read-only (record) access, ``'w'`` for - write-only (playback) access and ``'rw'`` for both. Since many sound cards - only allow one process to have the recorder or player open at a time, it is a - good idea to open the device only for the activity needed. Further, some - sound cards are half-duplex: they can be opened for reading or writing, but - not both at once. - - Note the unusual calling syntax: the *first* argument is optional, and the - second is required. This is a historical artifact for compatibility with the - older :mod:`linuxaudiodev` module which :mod:`ossaudiodev` supersedes. - - .. XXX it might also be motivated - by my unfounded-but-still-possibly-true belief that the default - audio device varies unpredictably across operating systems. -GW - - -.. function:: openmixer([device]) - - Open a mixer device and return an OSS mixer device object. *device* is the - mixer device filename to use. If it is not specified, this module first looks - in the environment variable :envvar:`MIXERDEV` for a device to use. If not - found, it falls back to :file:`/dev/mixer`. - - -.. _ossaudio-device-objects: - -Audio Device Objects --------------------- - -Before you can write to or read from an audio device, you must call three -methods in the correct order: - -#. :meth:`setfmt` to set the output format - -#. :meth:`channels` to set the number of channels - -#. :meth:`speed` to set the sample rate - -Alternately, you can use the :meth:`setparameters` method to set all three audio -parameters at once. This is more convenient, but may not be as flexible in all -cases. - -The audio device objects returned by :func:`.open` define the following methods -and (read-only) attributes: - - -.. method:: oss_audio_device.close() - - Explicitly close the audio device. When you are done writing to or reading from - an audio device, you should explicitly close it. A closed device cannot be used - again. - - -.. method:: oss_audio_device.fileno() - - Return the file descriptor associated with the device. - - -.. method:: oss_audio_device.read(size) - - Read *size* bytes from the audio input and return them as a Python string. - Unlike most Unix device drivers, OSS audio devices in blocking mode (the - default) will block :func:`read` until the entire requested amount of data is - available. - - -.. method:: oss_audio_device.write(data) - - Write a :term:`bytes-like object` *data* to the audio device and return the - number of bytes written. If the audio device is in blocking mode (the - default), the entire data is always written (again, this is different from - usual Unix device semantics). If the device is in non-blocking mode, some - data may not be written---see :meth:`writeall`. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. method:: oss_audio_device.writeall(data) - - Write a :term:`bytes-like object` *data* to the audio device: waits until - the audio device is able to accept data, writes as much data as it will - accept, and repeats until *data* has been completely written. If the device - is in blocking mode (the default), this has the same effect as - :meth:`write`; :meth:`writeall` is only useful in non-blocking mode. Has - no return value, since the amount of data written is always equal to the - amount of data supplied. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. versionchanged:: 3.2 - Audio device objects also support the context management protocol, i.e. they can - be used in a :keyword:`with` statement. - - -The following methods each map to exactly one :c:func:`ioctl` system call. The -correspondence is obvious: for example, :meth:`setfmt` corresponds to the -``SNDCTL_DSP_SETFMT`` ioctl, and :meth:`sync` to ``SNDCTL_DSP_SYNC`` (this can -be useful when consulting the OSS documentation). If the underlying -:c:func:`ioctl` fails, they all raise :exc:`OSError`. - - -.. method:: oss_audio_device.nonblock() - - Put the device into non-blocking mode. Once in non-blocking mode, there is no - way to return it to blocking mode. - - -.. method:: oss_audio_device.getfmts() - - Return a bitmask of the audio output formats supported by the soundcard. Some - of the formats supported by OSS are: - - +-------------------------+---------------------------------------------+ - | Format | Description | - +=========================+=============================================+ - | :const:`AFMT_MU_LAW` | a logarithmic encoding (used by Sun ``.au`` | - | | files and :file:`/dev/audio`) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_A_LAW` | a logarithmic encoding | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_IMA_ADPCM` | a 4:1 compressed format defined by the | - | | Interactive Multimedia Association | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U8` | Unsigned, 8-bit audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S16_LE` | Signed, 16-bit audio, little-endian byte | - | | order (as used by Intel processors) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S16_BE` | Signed, 16-bit audio, big-endian byte order | - | | (as used by 68k, PowerPC, Sparc) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S8` | Signed, 8 bit audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U16_LE` | Unsigned, 16-bit little-endian audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U16_BE` | Unsigned, 16-bit big-endian audio | - +-------------------------+---------------------------------------------+ - - Consult the OSS documentation for a full list of audio formats, and note that - most devices support only a subset of these formats. Some older devices only - support :const:`AFMT_U8`; the most common format used today is - :const:`AFMT_S16_LE`. - - -.. method:: oss_audio_device.setfmt(format) - - Try to set the current audio format to *format*---see :meth:`getfmts` for a - list. Returns the audio format that the device was set to, which may not be the - requested format. May also be used to return the current audio format---do this - by passing an "audio format" of :const:`AFMT_QUERY`. - - -.. method:: oss_audio_device.channels(nchannels) - - Set the number of output channels to *nchannels*. A value of 1 indicates - monophonic sound, 2 stereophonic. Some devices may have more than 2 channels, - and some high-end devices may not support mono. Returns the number of channels - the device was set to. - - -.. method:: oss_audio_device.speed(samplerate) - - Try to set the audio sampling rate to *samplerate* samples per second. Returns - the rate actually set. Most sound devices don't support arbitrary sampling - rates. Common rates are: - - +-------+-------------------------------------------+ - | Rate | Description | - +=======+===========================================+ - | 8000 | default rate for :file:`/dev/audio` | - +-------+-------------------------------------------+ - | 11025 | speech recording | - +-------+-------------------------------------------+ - | 22050 | | - +-------+-------------------------------------------+ - | 44100 | CD quality audio (at 16 bits/sample and 2 | - | | channels) | - +-------+-------------------------------------------+ - | 96000 | DVD quality audio (at 24 bits/sample) | - +-------+-------------------------------------------+ - - -.. method:: oss_audio_device.sync() - - Wait until the sound device has played every byte in its buffer. (This happens - implicitly when the device is closed.) The OSS documentation recommends closing - and re-opening the device rather than using :meth:`sync`. - - -.. method:: oss_audio_device.reset() - - Immediately stop playing or recording and return the device to a state where it - can accept commands. The OSS documentation recommends closing and re-opening - the device after calling :meth:`reset`. - - -.. method:: oss_audio_device.post() - - Tell the driver that there is likely to be a pause in the output, making it - possible for the device to handle the pause more intelligently. You might use - this after playing a spot sound effect, before waiting for user input, or before - doing disk I/O. - -The following convenience methods combine several ioctls, or one ioctl and some -simple calculations. - - -.. method:: oss_audio_device.setparameters(format, nchannels, samplerate[, strict=False]) - - Set the key audio sampling parameters---sample format, number of channels, and - sampling rate---in one method call. *format*, *nchannels*, and *samplerate* - should be as specified in the :meth:`setfmt`, :meth:`channels`, and - :meth:`speed` methods. If *strict* is true, :meth:`setparameters` checks to - see if each parameter was actually set to the requested value, and raises - :exc:`OSSAudioError` if not. Returns a tuple (*format*, *nchannels*, - *samplerate*) indicating the parameter values that were actually set by the - device driver (i.e., the same as the return values of :meth:`setfmt`, - :meth:`channels`, and :meth:`speed`). - - For example, :: - - (fmt, channels, rate) = dsp.setparameters(fmt, channels, rate) - - is equivalent to :: - - fmt = dsp.setfmt(fmt) - channels = dsp.channels(channels) - rate = dsp.rate(rate) - - -.. method:: oss_audio_device.bufsize() - - Returns the size of the hardware buffer, in samples. - - -.. method:: oss_audio_device.obufcount() - - Returns the number of samples that are in the hardware buffer yet to be played. - - -.. method:: oss_audio_device.obuffree() - - Returns the number of samples that could be queued into the hardware buffer to - be played without blocking. - -Audio device objects also support several read-only attributes: - - -.. attribute:: oss_audio_device.closed - - Boolean indicating whether the device has been closed. - - -.. attribute:: oss_audio_device.name - - String containing the name of the device file. - - -.. attribute:: oss_audio_device.mode - - The I/O mode for the file, either ``"r"``, ``"rw"``, or ``"w"``. - - -.. _mixer-device-objects: - -Mixer Device Objects --------------------- - -The mixer object provides two file-like methods: - - -.. method:: oss_mixer_device.close() - - This method closes the open mixer device file. Any further attempts to use the - mixer after this file is closed will raise an :exc:`OSError`. - - -.. method:: oss_mixer_device.fileno() - - Returns the file handle number of the open mixer device file. - -.. versionchanged:: 3.2 - Mixer objects also support the context management protocol. - - -The remaining methods are specific to audio mixing: - - -.. method:: oss_mixer_device.controls() - - This method returns a bitmask specifying the available mixer controls ("Control" - being a specific mixable "channel", such as :const:`SOUND_MIXER_PCM` or - :const:`SOUND_MIXER_SYNTH`). This bitmask indicates a subset of all available - mixer controls---the :const:`SOUND_MIXER_\*` constants defined at module level. - To determine if, for example, the current mixer object supports a PCM mixer, use - the following Python code:: - - mixer=ossaudiodev.openmixer() - if mixer.controls() & (1 << ossaudiodev.SOUND_MIXER_PCM): - # PCM is supported - ... code ... - - For most purposes, the :const:`SOUND_MIXER_VOLUME` (master volume) and - :const:`SOUND_MIXER_PCM` controls should suffice---but code that uses the mixer - should be flexible when it comes to choosing mixer controls. On the Gravis - Ultrasound, for example, :const:`SOUND_MIXER_VOLUME` does not exist. - - -.. method:: oss_mixer_device.stereocontrols() - - Returns a bitmask indicating stereo mixer controls. If a bit is set, the - corresponding control is stereo; if it is unset, the control is either - monophonic or not supported by the mixer (use in combination with - :meth:`controls` to determine which). - - See the code example for the :meth:`controls` function for an example of getting - data from a bitmask. - - -.. method:: oss_mixer_device.reccontrols() - - Returns a bitmask specifying the mixer controls that may be used to record. See - the code example for :meth:`controls` for an example of reading from a bitmask. - - -.. method:: oss_mixer_device.get(control) - - Returns the volume of a given mixer control. The returned volume is a 2-tuple - ``(left_volume,right_volume)``. Volumes are specified as numbers from 0 - (silent) to 100 (full volume). If the control is monophonic, a 2-tuple is still - returned, but both volumes are the same. - - Raises :exc:`OSSAudioError` if an invalid control is specified, or - :exc:`OSError` if an unsupported control is specified. - - -.. method:: oss_mixer_device.set(control, (left, right)) - - Sets the volume for a given mixer control to ``(left,right)``. ``left`` and - ``right`` must be ints and between 0 (silent) and 100 (full volume). On - success, the new volume is returned as a 2-tuple. Note that this may not be - exactly the same as the volume specified, because of the limited resolution of - some soundcard's mixers. - - Raises :exc:`OSSAudioError` if an invalid mixer control was specified, or if the - specified volumes were out-of-range. - - -.. method:: oss_mixer_device.get_recsrc() - - This method returns a bitmask indicating which control(s) are currently being - used as a recording source. - - -.. method:: oss_mixer_device.set_recsrc(bitmask) - - Call this function to specify a recording source. Returns a bitmask indicating - the new recording source (or sources) if successful; raises :exc:`OSError` if an - invalid source was specified. To set the current recording source to the - microphone input:: - - mixer.setrecsrc (1 << ossaudiodev.SOUND_MIXER_MIC) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 93af07ae5ac10f..60791725c2323d 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -88,6 +88,17 @@ Opening a file:: '#!/bin/bash\n' +Exceptions +---------- + +.. exception:: UnsupportedOperation + + An exception inheriting :exc:`NotImplementedError` that is raised when an + unsupported operation is called on a path object. + + .. versionadded:: 3.13 + + .. _pure-paths: Pure paths @@ -292,6 +303,13 @@ Methods and properties Pure paths provide the following methods and properties: +.. attribute:: PurePath.pathmod + + The implementation of the :mod:`os.path` module used for low-level path + operations: either :mod:`posixpath` or :mod:`ntpath`. + + .. versionadded:: 3.13 + .. attribute:: PurePath.drive A string representing the drive letter or name, if any:: @@ -421,7 +439,7 @@ Pure paths provide the following methods and properties: .. attribute:: PurePath.suffix - The file extension of the final component, if any:: + The last dot-separated portion of the final component, if any:: >>> PurePosixPath('my/library/setup.py').suffix '.py' @@ -430,10 +448,11 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').suffix '' + This is commonly called the file extension. .. attribute:: PurePath.suffixes - A list of the path's file extensions:: + A list of the path's suffixes, often called file extensions:: >>> PurePosixPath('my/library.tar.gar').suffixes ['.tar', '.gar'] @@ -546,7 +565,7 @@ Pure paths provide the following methods and properties: PureWindowsPath('c:/Program Files') -.. method:: PurePath.match(pattern) +.. method:: PurePath.match(pattern, *, case_sensitive=None) Match this path against the provided glob-style pattern. Return ``True`` if matching is successful, ``False`` otherwise. @@ -569,6 +588,16 @@ Pure paths provide the following methods and properties: >>> PurePath('a/b.py').match('/*.py') False + The *pattern* may be another path object; this speeds up matching the same + pattern against multiple files:: + + >>> pattern = PurePath('*.py') + >>> PurePath('a/b.py').match(pattern) + True + + .. versionchanged:: 3.12 + Accepts an object implementing the :class:`os.PathLike` interface. + As with other methods, case-sensitivity follows platform defaults:: >>> PurePosixPath('b.py').match('*.PY') @@ -576,6 +605,15 @@ Pure paths provide the following methods and properties: >>> PureWindowsPath('b.py').match('*.PY') True + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + Support for the recursive wildcard "``**``" was added. In previous + versions, it acted like the non-recursive wildcard "``*``". + .. method:: PurePath.relative_to(other, walk_up=False) @@ -615,8 +653,8 @@ Pure paths provide the following methods and properties: are present in the path; call :meth:`~Path.resolve` first if necessary to resolve symlinks. - .. versionadded:: 3.12 - The *walk_up* argument (old behavior is the same as ``walk_up=False``). + .. versionchanged:: 3.12 + The *walk_up* parameter was added (old behavior is the same as ``walk_up=False``). .. deprecated-removed:: 3.12 3.14 @@ -736,6 +774,11 @@ calls on path objects. There are three ways to instantiate concrete paths: *pathsegments* is specified similarly to :class:`PurePath`. + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` on Windows. In previous versions, + :exc:`NotImplementedError` was raised instead. + + .. class:: WindowsPath(*pathsegments) A subclass of :class:`Path` and :class:`PureWindowsPath`, this class @@ -746,6 +789,11 @@ calls on path objects. There are three ways to instantiate concrete paths: *pathsegments* is specified similarly to :class:`PurePath`. + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` on non-Windows platforms. In previous + versions, :exc:`NotImplementedError` was raised instead. + + You can only instantiate the class flavour that corresponds to your system (allowing system calls on non-compatible path flavours could lead to bugs or failures in your application):: @@ -762,7 +810,7 @@ bugs or failures in your application):: File "", line 1, in File "pathlib.py", line 798, in __new__ % (cls.__name__,)) - NotImplementedError: cannot instantiate 'WindowsPath' on your system + UnsupportedOperation: cannot instantiate 'WindowsPath' on your system Methods @@ -805,6 +853,42 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 +.. classmethod:: Path.from_uri(uri) + + Return a new path object from parsing a 'file' URI conforming to + :rfc:`8089`. For example:: + + >>> p = Path.from_uri('file:///etc/hosts') + PosixPath('/etc/hosts') + + On Windows, DOS device and UNC paths may be parsed from URIs:: + + >>> p = Path.from_uri('file:///c:/windows') + WindowsPath('c:/windows') + >>> p = Path.from_uri('file://server/share') + WindowsPath('//server/share') + + Several variant forms are supported:: + + >>> p = Path.from_uri('file:////server/share') + WindowsPath('//server/share') + >>> p = Path.from_uri('file://///server/share') + WindowsPath('//server/share') + >>> p = Path.from_uri('file:c:/windows') + WindowsPath('c:/windows') + >>> p = Path.from_uri('file:/c|/windows') + WindowsPath('c:/windows') + + :exc:`ValueError` is raised if the URI does not start with ``file:``, or + the parsed path isn't absolute. + + :func:`os.fsdecode` is used to decode percent-escaped byte sequences, and + so file URIs are not portable across machines with different + :ref:`filesystem encodings `. + + .. versionadded:: 3.13 + + .. method:: Path.stat(*, follow_symlinks=True) Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. @@ -880,7 +964,7 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 -.. method:: Path.glob(pattern, *, case_sensitive=None) +.. method:: Path.glob(pattern, *, case_sensitive=None, follow_symlinks=None) Glob the given relative *pattern* in the directory represented by this path, yielding all matching files (of any kind):: @@ -901,14 +985,23 @@ call fails (for example because the path doesn't exist). PosixPath('setup.py'), PosixPath('test_pathlib.py')] + .. note:: + Using the "``**``" pattern in large directory trees may consume + an inordinate amount of time. + + .. tip:: + Set *follow_symlinks* to ``True`` or ``False`` to improve performance + of recursive globbing. + By default, or when the *case_sensitive* keyword-only argument is set to ``None``, this method matches paths using platform-specific casing rules: typically, case-sensitive on POSIX, and case-insensitive on Windows. Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. + By default, or when the *follow_symlinks* keyword-only argument is set to + ``None``, this method follows symlinks except when expanding "``**``" + wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or + ``False`` to treat all symlinks as files. .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob @@ -916,32 +1009,62 @@ call fails (for example because the path doesn't exist). Return only directories if *pattern* ends with a pathname components separator (:data:`~os.sep` or :data:`~os.altsep`). - .. versionadded:: 3.12 - The *case_sensitive* argument. + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + + .. versionchanged:: 3.13 + Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a + future Python release, patterns with this ending will match both files + and directories. Add a trailing slash to match only directories. -.. method:: Path.group() +.. method:: Path.group(*, follow_symlinks=True) - Return the name of the group owning the file. :exc:`KeyError` is raised + Return the name of the group owning the file. :exc:`KeyError` is raised if the file's gid isn't found in the system database. + This method normally follows symlinks; to get the group of the symlink, add + the argument ``follow_symlinks=False``. + + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + -.. method:: Path.is_dir() +.. method:: Path.is_dir(*, follow_symlinks=True) - Return ``True`` if the path points to a directory (or a symbolic link - pointing to a directory), ``False`` if it points to another kind of file. + Return ``True`` if the path points to a directory, ``False`` if it points + to another kind of file. ``False`` is also returned if the path doesn't exist or is a broken symlink; other errors (such as permission errors) are propagated. + This method normally follows symlinks; to exclude symlinks to directories, + add the argument ``follow_symlinks=False``. -.. method:: Path.is_file() + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + + +.. method:: Path.is_file(*, follow_symlinks=True) - Return ``True`` if the path points to a regular file (or a symbolic link - pointing to a regular file), ``False`` if it points to another kind of file. + Return ``True`` if the path points to a regular file, ``False`` if it + points to another kind of file. ``False`` is also returned if the path doesn't exist or is a broken symlink; other errors (such as permission errors) are propagated. + This method normally follows symlinks; to exclude symlinks, add the + argument ``follow_symlinks=False``. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + .. method:: Path.is_junction() @@ -1177,11 +1300,21 @@ call fails (for example because the path doesn't exist). '#!/usr/bin/env python3\n' -.. method:: Path.owner() +.. method:: Path.owner(*, follow_symlinks=True) - Return the name of the user owning the file. :exc:`KeyError` is raised + Return the name of the user owning the file. :exc:`KeyError` is raised if the file's uid isn't found in the system database. + This method normally follows symlinks; to get the owner of the symlink, add + the argument ``follow_symlinks=False``. + + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if the :mod:`pwd` module is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + .. method:: Path.read_bytes() @@ -1196,7 +1329,7 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 -.. method:: Path.read_text(encoding=None, errors=None) +.. method:: Path.read_text(encoding=None, errors=None, newline=None) Return the decoded contents of the pointed-to file as a string:: @@ -1211,6 +1344,8 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 + .. versionchanged:: 3.13 + The *newline* parameter was added. .. method:: Path.readlink() @@ -1224,6 +1359,10 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.9 + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.readlink` is not + available. In previous versions, :exc:`NotImplementedError` was raised. + .. method:: Path.rename(target) @@ -1295,16 +1434,20 @@ call fails (for example because the path doesn't exist). >>> p.resolve() PosixPath('/home/antoine/pathlib/setup.py') - If the path doesn't exist and *strict* is ``True``, :exc:`FileNotFoundError` - is raised. If *strict* is ``False``, the path is resolved as far as possible - and any remainder is appended without checking whether it exists. If an - infinite loop is encountered along the resolution path, :exc:`RuntimeError` - is raised. + If a path doesn't exist or a symlink loop is encountered, and *strict* is + ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is + resolved as far as possible and any remainder is appended without checking + whether it exists. - .. versionadded:: 3.6 - The *strict* argument (pre-3.6 behavior is strict). + .. versionchanged:: 3.6 + The *strict* parameter was added (pre-3.6 behavior is strict). -.. method:: Path.rglob(pattern, *, case_sensitive=None) + .. versionchanged:: 3.13 + Symlink loops are treated like other errors: :exc:`OSError` is raised in + strict mode, and no exception is raised in non-strict mode. In previous + versions, :exc:`RuntimeError` is raised no matter the value of *strict*. + +.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) Glob the given relative *pattern* recursively. This is like calling :func:`Path.glob` with "``**/``" added in front of the *pattern*, where @@ -1322,14 +1465,22 @@ call fails (for example because the path doesn't exist). typically, case-sensitive on POSIX, and case-insensitive on Windows. Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + By default, or when the *follow_symlinks* keyword-only argument is set to + ``None``, this method follows symlinks except when expanding "``**``" + wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or + ``False`` to treat all symlinks as files. + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob .. versionchanged:: 3.11 Return only directories if *pattern* ends with a pathname components separator (:data:`~os.sep` or :data:`~os.altsep`). - .. versionadded:: 3.12 - The *case_sensitive* argument. + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. .. method:: Path.rmdir() @@ -1378,6 +1529,11 @@ call fails (for example because the path doesn't exist). The order of arguments (link, target) is the reverse of :func:`os.symlink`'s. + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + .. method:: Path.hardlink_to(target) Make this path a hard link to the same file as *target*. @@ -1388,6 +1544,10 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.10 + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.link` is not + available. In previous versions, :exc:`NotImplementedError` was raised. + .. method:: Path.touch(mode=0o666, exist_ok=True) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index ef52370bff8058..2495dcf50bb17f 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -175,8 +175,8 @@ slightly different way: .. function:: pm() - Enter post-mortem debugging of the traceback found in - :data:`sys.last_traceback`. + Enter post-mortem debugging of the exception found in + :data:`sys.last_exc`. The ``run*`` functions and :func:`set_trace` are aliases for instantiating the @@ -252,6 +252,10 @@ change a variable or call a function. When an exception occurs in such a statement, the exception name is printed but the debugger's state is not changed. +.. versionchanged:: 3.13 + Expressions/Statements whose prefix is a pdb command are now correctly + identified and executed. + The debugger supports :ref:`aliases `. Aliases can have parameters which allows one a certain level of adaptability to the context under examination. @@ -566,17 +570,34 @@ can be overridden by the local file. Start an interactive interpreter (using the :mod:`code` module) whose global namespace contains all the (global and local) names found in the current - scope. + scope. Use ``exit()`` or ``quit()`` to exit the interpreter and return to + the debugger. + + .. note:: + + Because interact creates a new global namespace with the current global + and local namespace for execution, assignment to variables will not + affect the original namespaces. + However, modification to the mutable objects will be reflected in the + original namespaces. .. versionadded:: 3.2 + .. versionadded:: 3.13 + ``exit()`` and ``quit()`` can be used to exit :pdbcmd:`interact` + command. + + .. versionchanged:: 3.13 + :pdbcmd:`interact` directs its output to the debugger's + output channel rather than :data:`sys.stderr`. + .. _debugger-aliases: .. pdbcommand:: alias [name [command]] Create an alias called *name* that executes *command*. The *command* must *not* be enclosed in quotes. Replaceable parameters can be indicated by - ``%1``, ``%2``, and so on, while ``%*`` is replaced by all the parameters. + ``%1``, ``%2``, ... and ``%9``, while ``%*`` is replaced by all the parameters. If *command* is omitted, the current alias for *name* is shown. If no arguments are given, all aliases are listed. @@ -639,6 +660,55 @@ can be overridden by the local file. Print the return value for the last return of the current function. +.. pdbcommand:: exceptions [excnumber] + + List or jump between chained exceptions. + + When using ``pdb.pm()`` or ``Pdb.post_mortem(...)`` with a chained exception + instead of a traceback, it allows the user to move between the + chained exceptions using ``exceptions`` command to list exceptions, and + ``exception `` to switch to that exception. + + + Example:: + + def out(): + try: + middle() + except Exception as e: + raise ValueError("reraise middle() error") from e + + def middle(): + try: + return inner(0) + except Exception as e: + raise ValueError("Middle fail") + + def inner(x): + 1 / x + + out() + + calling ``pdb.pm()`` will allow to move between exceptions:: + + > example.py(5)out() + -> raise ValueError("reraise middle() error") from e + + (Pdb) exceptions + 0 ZeroDivisionError('division by zero') + 1 ValueError('Middle fail') + > 2 ValueError('reraise middle() error') + + (Pdb) exceptions 0 + > example.py(16)inner() + -> 1 / x + + (Pdb) up + > example.py(10)middle() + -> return inner(0) + + .. versionadded:: 3.13 + .. rubric:: Footnotes .. [1] Whether a frame is considered to originate in a certain module diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 79476b04cd914d..93387fb0b45038 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -125,7 +125,7 @@ Data stream format The data format used by :mod:`pickle` is Python-specific. This has the advantage that there are no restrictions imposed by external standards such as -JSON or XDR (which can't represent pointer sharing); however it means that +JSON (which can't represent pointer sharing); however it means that non-Python programs may not be able to reconstruct pickled Python objects. By default, the :mod:`pickle` data format uses a relatively compact binary @@ -494,7 +494,8 @@ What can be pickled and unpickled? The following types can be pickled: -* ``None``, ``True``, and ``False``; +* built-in constants (``None``, ``True``, ``False``, ``Ellipsis``, and + ``NotImplemented``); * integers, floating-point numbers, complex numbers; diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 480f4a6d320815..41930f8cbe8412 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -17,6 +17,8 @@ are useful for Python core developers who are working on the :mod:`pickle`; ordinary users of the :mod:`pickle` module probably won't find the :mod:`pickletools` module relevant. +.. _pickletools-cli: + Command line usage ------------------ @@ -51,24 +53,24 @@ Command line options .. program:: pickletools -.. cmdoption:: -a, --annotate +.. option:: -a, --annotate Annotate each line with a short opcode description. -.. cmdoption:: -o, --output= +.. option:: -o, --output= Name of a file where the output should be written. -.. cmdoption:: -l, --indentlevel= +.. option:: -l, --indentlevel= The number of blanks by which to indent a new MARK level. -.. cmdoption:: -m, --memo +.. option:: -m, --memo When multiple objects are disassembled, preserve memo between disassemblies. -.. cmdoption:: -p, --preamble= +.. option:: -p, --preamble= When more than one pickle file are specified, print given preamble before each disassembly. diff --git a/Doc/library/pipes.rst b/Doc/library/pipes.rst deleted file mode 100644 index 471ae0dbc9768a..00000000000000 --- a/Doc/library/pipes.rst +++ /dev/null @@ -1,103 +0,0 @@ -:mod:`pipes` --- Interface to shell pipelines -============================================= - -.. module:: pipes - :platform: Unix - :synopsis: A Python interface to Unix shell pipelines. - :deprecated: - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/pipes.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`pipes` module is deprecated - (see :pep:`PEP 594 <594#pipes>` for details). - Please use the :mod:`subprocess` module instead. - --------------- - -The :mod:`pipes` module defines a class to abstract the concept of a *pipeline* ---- a sequence of converters from one file to another. - -Because the module uses :program:`/bin/sh` command lines, a POSIX or compatible -shell for :func:`os.system` and :func:`os.popen` is required. - -.. availability:: Unix, not VxWorks. - -The :mod:`pipes` module defines the following class: - - -.. class:: Template() - - An abstraction of a pipeline. - -Example:: - - >>> import pipes - >>> t = pipes.Template() - >>> t.append('tr a-z A-Z', '--') - >>> f = t.open('pipefile', 'w') - >>> f.write('hello world') - >>> f.close() - >>> open('pipefile').read() - 'HELLO WORLD' - - -.. _template-objects: - -Template Objects ----------------- - -Template objects following methods: - - -.. method:: Template.reset() - - Restore a pipeline template to its initial state. - - -.. method:: Template.clone() - - Return a new, equivalent, pipeline template. - - -.. method:: Template.debug(flag) - - If *flag* is true, turn debugging on. Otherwise, turn debugging off. When - debugging is on, commands to be executed are printed, and the shell is given - ``set -x`` command to be more verbose. - - -.. method:: Template.append(cmd, kind) - - Append a new action at the end. The *cmd* variable must be a valid bourne shell - command. The *kind* variable consists of two letters. - - The first letter can be either of ``'-'`` (which means the command reads its - standard input), ``'f'`` (which means the commands reads a given file on the - command line) or ``'.'`` (which means the commands reads no input, and hence - must be first.) - - Similarly, the second letter can be either of ``'-'`` (which means the command - writes to standard output), ``'f'`` (which means the command writes a file on - the command line) or ``'.'`` (which means the command does not write anything, - and hence must be last.) - - -.. method:: Template.prepend(cmd, kind) - - Add a new action at the beginning. See :meth:`append` for explanations of the - arguments. - - -.. method:: Template.open(file, mode) - - Return a file-like object, open to *file*, but read from or written to by the - pipeline. Note that only one of ``'r'``, ``'w'`` may be given. - - -.. method:: Template.copy(infile, outfile) - - Copy *infile* to *outfile* through the pipe. - diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 69c4dfc422c98e..ec2a7ebd5d6e0b 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -46,7 +46,7 @@ Cross Platform universal files containing multiple architectures. To get at the "64-bitness" of the current interpreter, it is more - reliable to query the :attr:`sys.maxsize` attribute:: + reliable to query the :data:`sys.maxsize` attribute:: is_64bits = sys.maxsize > 2**32 diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index d8618ce9b60bba..943eb21f6eec02 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -77,21 +77,14 @@ The :mod:`poplib` module provides two classes: .. versionchanged:: 3.4 The class now supports hostname check with :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. + :const:`ssl.HAS_SNI`). .. versionchanged:: 3.9 If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket. .. versionchanged:: 3.12 - The deprecated *keyfile* and *certfile* parameters have been removed. + The deprecated *keyfile* and *certfile* parameters have been removed. One exception is defined as an attribute of the :mod:`poplib` module: @@ -155,7 +148,7 @@ A :class:`POP3` instance has the following methods: .. method:: POP3.pass_(password) Send password, response includes message count and mailbox size. Note: the - mailbox on the server is locked until :meth:`~poplib.quit` is called. + mailbox on the server is locked until :meth:`~POP3.quit` is called. .. method:: POP3.apop(user, secret) @@ -247,7 +240,7 @@ A :class:`POP3` instance has the following methods: This method supports hostname checking via :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). + :const:`ssl.HAS_SNI`). .. versionadded:: 3.4 diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 0413f9d02a8d57..5871574b442667 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -11,6 +11,8 @@ This module provides access to operating system functionality that is standardized by the C Standard and the POSIX standard (a thinly disguised Unix interface). +.. availability:: Unix. + .. index:: pair: module; os **Do not import this module directly.** Instead, import the module :mod:`os`, diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index d8269ef48cb36a..e883acd67d6c72 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -45,7 +45,7 @@ The :mod:`pprint` module defines one class: several keyword parameters. *stream* (default ``sys.stdout``) is a :term:`file-like object` to - which the output will be written by calling its :meth:`write` method. + which the output will be written by calling its :meth:`!write` method. If both *stream* and ``sys.stdout`` are ``None``, then :meth:`~PrettyPrinter.pprint` silently returns. diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 723f927135a0f4..cc059b66fcb84b 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -82,8 +82,8 @@ the following:: The first line indicates that 214 calls were monitored. Of those calls, 207 were :dfn:`primitive`, meaning that the call was not induced via recursion. The -next line: ``Ordered by: cumulative time``, indicates that the text string in the -far right column was used to sort the output. The column headings include: +next line: ``Ordered by: cumulative time`` indicates the output is sorted +by the ``cumtime`` values. The column headings include: ncalls for the number of calls. @@ -121,6 +121,8 @@ results to a file by specifying a filename to the :func:`run` function:: The :class:`pstats.Stats` class reads profile results from a file and formats them in various ways. +.. _profile-cli: + The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to profile another script. For example:: @@ -133,11 +135,11 @@ the output by. This only applies when ``-o`` is not supplied. ``-m`` specifies that a module is being profiled instead of a script. - .. versionadded:: 3.7 - Added the ``-m`` option to :mod:`cProfile`. +.. versionadded:: 3.7 + Added the ``-m`` option to :mod:`cProfile`. - .. versionadded:: 3.8 - Added the ``-m`` option to :mod:`profile`. +.. versionadded:: 3.8 + Added the ``-m`` option to :mod:`profile`. The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods for manipulating and printing the data saved into a profile results file:: diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 7f4da41e93802d..af9378464edb9f 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -16,6 +16,8 @@ The :mod:`pty` module defines operations for handling the pseudo-terminal concept: starting another process and being able to write to and read from its controlling terminal programmatically. +.. availability:: Unix. + Pseudo-terminal handling is highly platform dependent. This code is mainly tested on Linux, FreeBSD, and macOS (it is supposed to work on other POSIX platforms but it's not been thoroughly tested). @@ -71,7 +73,7 @@ The :mod:`pty` module defines the following functions: Return the exit status value from :func:`os.waitpid` on the child process. - :func:`waitstatus_to_exitcode` can be used to convert the exit status into + :func:`os.waitstatus_to_exitcode` can be used to convert the exit status into an exit code. .. audit-event:: pty.spawn argv pty.spawn diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index 7cafc66fd7e93c..dbe68cd14ec4d4 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -10,7 +10,7 @@ This module provides access to the Unix user account and password database. It is available on all Unix versions. -.. include:: ../includes/wasm-notavail.rst +.. availability:: Unix, not Emscripten, not WASI. Password database entries are reported as a tuple-like object, whose attributes correspond to the members of the ``passwd`` structure (Attribute field below, @@ -39,16 +39,13 @@ raised if the entry asked for cannot be found. .. note:: - .. index:: pair: module; crypt - In traditional Unix the field ``pw_passwd`` usually contains a password - encrypted with a DES derived algorithm (see module :mod:`crypt`). However most + encrypted with a DES derived algorithm. However most modern unices use a so-called *shadow password* system. On those unices the *pw_passwd* field only contains an asterisk (``'*'``) or the letter ``'x'`` where the encrypted password is stored in a file :file:`/etc/shadow` which is not world readable. Whether the *pw_passwd* field contains anything useful is - system-dependent. If available, the :mod:`spwd` module should be used where - access to the encrypted password is required. + system-dependent. It defines the following items: @@ -72,7 +69,3 @@ It defines the following items: Module :mod:`grp` An interface to the group database, similar to this. - - Module :mod:`spwd` - An interface to the shadow password database, similar to this. - diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 69b93a3bdfcb26..38c416f9ad0305 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -125,6 +125,7 @@ byte-code cache files in the directory containing the source code. This option is useful when the ``.pycs`` are kept up to date by some system external to Python like a build system. +.. _py_compile-cli: Command-Line Interface ---------------------- @@ -138,13 +139,13 @@ not be compiled. .. program:: python -m py_compile -.. cmdoption:: ... - - +.. option:: ... + - Positional arguments are files to compile. If ``-`` is the only parameter, the list of files is taken from standard input. -.. cmdoption:: -q, --quiet +.. option:: -q, --quiet Suppress errors output. diff --git a/Doc/library/python.rst b/Doc/library/python.rst index f39613f572884f..610435999d9f48 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -12,6 +12,7 @@ overview: .. toctree:: sys.rst + sys.monitoring.rst sysconfig.rst builtins.rst __main__.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index b7510b93d75427..251ec8ca0021a6 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -176,7 +176,7 @@ The special characters are: ``x*+``, ``x++`` and ``x?+`` are equivalent to ``(?>x*)``, ``(?>x+)`` and ``(?>x?)`` correspondingly. - .. versionadded:: 3.11 + .. versionadded:: 3.11 .. index:: single: {} (curly brackets); in regular expressions @@ -501,6 +501,8 @@ The special characters are: in the ASCII range (``b'\x00'``-``b'\x7f'``). +.. _re-special-sequences: + The special sequences consist of ``'\'`` and a character from the list below. If the ordinary character is not an ASCII digit or an ASCII letter, then the resulting RE will match the second character. For example, ``\$`` matches the @@ -632,8 +634,8 @@ character ``'$'``. single: \x; in regular expressions single: \\; in regular expressions -Most of the standard escapes supported by Python string literals are also -accepted by the regular expression parser:: +Most of the :ref:`escape sequences ` supported by Python +string literals are also accepted by the regular expression parser:: \a \b \f \n \N \r \t \u @@ -658,7 +660,7 @@ three digits in length. Unknown escapes consisting of ``'\'`` and an ASCII letter now are errors. .. versionchanged:: 3.8 - The ``'\N{name}'`` escape sequence has been added. As in string literals, + The :samp:`'\\N\\{{name}\\}'` escape sequence has been added. As in string literals, it expands to the named Unicode character (e.g. ``'\N{EM DASH}'``). @@ -779,6 +781,17 @@ Flags Corresponds to the inline flag ``(?s)``. +.. data:: U + UNICODE + + In Python 2, this flag made :ref:`special sequences ` + include Unicode characters in matches. Since Python 3, Unicode characters + are matched by default. + + See :const:`A` for restricting matching on ASCII characters instead. + + This flag is only kept for backward compatibility. + .. data:: X VERBOSE @@ -843,18 +856,17 @@ Functions .. function:: search(pattern, string, flags=0) Scan through *string* looking for the first location where the regular expression - *pattern* produces a match, and return a corresponding :ref:`match object - `. Return ``None`` if no position in the string matches the - pattern; note that this is different from finding a zero-length match at some - point in the string. + *pattern* produces a match, and return a corresponding :class:`~re.Match`. Return + ``None`` if no position in the string matches the pattern; note that this is + different from finding a zero-length match at some point in the string. .. function:: match(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular - expression *pattern*, return a corresponding :ref:`match object - `. Return ``None`` if the string does not match the pattern; - note that this is different from a zero-length match. + expression *pattern*, return a corresponding :class:`~re.Match`. Return + ``None`` if the string does not match the pattern; note that this is + different from a zero-length match. Note that even in :const:`MULTILINE` mode, :func:`re.match` will only match at the beginning of the string and not at the beginning of each line. @@ -866,9 +878,8 @@ Functions .. function:: fullmatch(pattern, string, flags=0) If the whole *string* matches the regular expression *pattern*, return a - corresponding :ref:`match object `. Return ``None`` if the - string does not match the pattern; note that this is different from a - zero-length match. + corresponding :class:`~re.Match`. Return ``None`` if the string does not match + the pattern; note that this is different from a zero-length match. .. versionadded:: 3.4 @@ -885,7 +896,7 @@ Functions ['Words', 'words', 'words', ''] >>> re.split(r'(\W+)', 'Words, words, words.') ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split(r'\W+', 'Words, words, words.', 1) + >>> re.split(r'\W+', 'Words, words, words.', maxsplit=1) ['Words', 'words, words.'] >>> re.split('[a-f]+', '0a3B9', flags=re.IGNORECASE) ['0', '3', '9'] @@ -916,6 +927,11 @@ Functions .. versionchanged:: 3.7 Added support of splitting on a pattern that could match an empty string. + .. deprecated:: 3.13 + Passing *maxsplit* and *flags* as positional arguments is deprecated. + In future Python versions they will be + :ref:`keyword-only parameters `. + .. function:: findall(pattern, string, flags=0) @@ -941,7 +957,7 @@ Functions .. function:: finditer(pattern, string, flags=0) - Return an :term:`iterator` yielding :ref:`match objects ` over + Return an :term:`iterator` yielding :class:`~re.Match` objects over all non-overlapping matches for the RE *pattern* in *string*. The *string* is scanned left-to-right, and matches are returned in the order found. Empty matches are included in the result. @@ -969,8 +985,8 @@ Functions 'static PyObject*\npy_myfunc(void)\n{' If *repl* is a function, it is called for every non-overlapping occurrence of - *pattern*. The function takes a single :ref:`match object ` - argument, and returns the replacement string. For example:: + *pattern*. The function takes a single :class:`~re.Match` argument, and returns + the replacement string. For example:: >>> def dashrepl(matchobj): ... if matchobj.group(0) == '-': return ' ' @@ -981,7 +997,7 @@ Functions >>> re.sub(r'\sAND\s', ' & ', 'Baked Beans And Spam', flags=re.IGNORECASE) 'Baked Beans & Spam' - The pattern may be a string or a :ref:`pattern object `. + The pattern may be a string or a :class:`~re.Pattern`. The optional argument *count* is the maximum number of pattern occurrences to be replaced; *count* must be a non-negative integer. If omitted or zero, all @@ -1014,8 +1030,6 @@ Functions .. versionchanged:: 3.7 Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now are errors. - - .. versionchanged:: 3.7 Empty matches for the pattern are replaced when adjacent to a previous non-empty match. @@ -1024,18 +1038,17 @@ Functions In :class:`bytes` replacement strings, group *name* can only contain bytes in the ASCII range (``b'\x00'``-``b'\x7f'``). + .. deprecated:: 3.13 + Passing *count* and *flags* as positional arguments is deprecated. + In future Python versions they will be + :ref:`keyword-only parameters `. + .. function:: subn(pattern, repl, string, count=0, flags=0) Perform the same operation as :func:`sub`, but return a tuple ``(new_string, number_of_subs_made)``. - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - .. function:: escape(pattern) @@ -1116,16 +1129,20 @@ Exceptions Regular Expression Objects -------------------------- -Compiled regular expression objects support the following methods and -attributes: +.. class:: Pattern + + Compiled regular expression object returned by :func:`re.compile`. + + .. versionchanged:: 3.9 + :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. + See :ref:`types-genericalias`. .. method:: Pattern.search(string[, pos[, endpos]]) Scan through *string* looking for the first location where this regular - expression produces a match, and return a corresponding :ref:`match object - `. Return ``None`` if no position in the string matches the - pattern; note that this is different from finding a zero-length match at some - point in the string. + expression produces a match, and return a corresponding :class:`~re.Match`. + Return ``None`` if no position in the string matches the pattern; note that + this is different from finding a zero-length match at some point in the string. The optional second parameter *pos* gives an index in the string where the search is to start; it defaults to ``0``. This is not completely equivalent to @@ -1149,9 +1166,9 @@ attributes: .. method:: Pattern.match(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular - expression, return a corresponding :ref:`match object `. - Return ``None`` if the string does not match the pattern; note that this is - different from a zero-length match. + expression, return a corresponding :class:`~re.Match`. Return ``None`` if the + string does not match the pattern; note that this is different from a + zero-length match. The optional *pos* and *endpos* parameters have the same meaning as for the :meth:`~Pattern.search` method. :: @@ -1168,8 +1185,8 @@ attributes: .. method:: Pattern.fullmatch(string[, pos[, endpos]]) If the whole *string* matches this regular expression, return a corresponding - :ref:`match object `. Return ``None`` if the string does not - match the pattern; note that this is different from a zero-length match. + :class:`~re.Match`. Return ``None`` if the string does not match the pattern; + note that this is different from a zero-length match. The optional *pos* and *endpos* parameters have the same meaning as for the :meth:`~Pattern.search` method. :: @@ -1255,8 +1272,13 @@ when there is no match, you can test whether there was a match with a simple if match: process(match) -Match objects support the following methods and attributes: +.. class:: Match + + Match object returned by successful ``match``\ es and ``search``\ es. + .. versionchanged:: 3.9 + :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. + See :ref:`types-genericalias`. .. method:: Match.expand(template) @@ -1520,14 +1542,14 @@ Simulating scanf() .. index:: single: scanf() -Python does not currently have an equivalent to :c:func:`scanf`. Regular +Python does not currently have an equivalent to :c:func:`!scanf`. Regular expressions are generally more powerful, though also more verbose, than -:c:func:`scanf` format strings. The table below offers some more-or-less -equivalent mappings between :c:func:`scanf` format tokens and regular +:c:func:`!scanf` format strings. The table below offers some more-or-less +equivalent mappings between :c:func:`!scanf` format tokens and regular expressions. +--------------------------------+---------------------------------------------+ -| :c:func:`scanf` Token | Regular Expression | +| :c:func:`!scanf` Token | Regular Expression | +================================+=============================================+ | ``%c`` | ``.`` | +--------------------------------+---------------------------------------------+ @@ -1552,7 +1574,7 @@ To extract the filename and numbers from a string like :: /usr/sbin/sendmail - 0 errors, 4 warnings -you would use a :c:func:`scanf` format like :: +you would use a :c:func:`!scanf` format like :: %s - %d errors, %d warnings @@ -1643,7 +1665,7 @@ because the address has spaces, our splitting pattern, in it: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> [re.split(":? ", entry, 3) for entry in entries] + >>> [re.split(":? ", entry, maxsplit=3) for entry in entries] [['Ross', 'McFluff', '834.345.1254', '155 Elm Street'], ['Ronald', 'Heathmore', '892.345.3428', '436 Finley Avenue'], ['Frank', 'Burger', '925.541.7625', '662 South Dogwood Way'], @@ -1656,7 +1678,7 @@ house number from the street name: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> [re.split(":? ", entry, 4) for entry in entries] + >>> [re.split(":? ", entry, maxsplit=4) for entry in entries] [['Ross', 'McFluff', '834.345.1254', '155', 'Elm Street'], ['Ronald', 'Heathmore', '892.345.3428', '436', 'Finley Avenue'], ['Frank', 'Burger', '925.541.7625', '662', 'South Dogwood Way'], @@ -1700,10 +1722,10 @@ Finding all Adverbs and their Positions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If one wants more information about all matches of a pattern than the matched -text, :func:`finditer` is useful as it provides :ref:`match objects -` instead of strings. Continuing with the previous example, if -a writer wanted to find all of the adverbs *and their positions* in -some text, they would use :func:`finditer` in the following manner:: +text, :func:`finditer` is useful as it provides :class:`~re.Match` objects +instead of strings. Continuing with the previous example, if a writer wanted +to find all of the adverbs *and their positions* in some text, they would use +:func:`finditer` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." >>> for m in re.finditer(r"\w+ly\b", text): diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 8fb0eca8df74d8..2e0f45ced30b9c 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -27,16 +27,15 @@ Readline library in general. .. note:: The underlying Readline library API may be implemented by - the ``libedit`` library instead of GNU readline. + the ``editline`` (``libedit``) library instead of GNU readline. On macOS the :mod:`readline` module detects which library is being used at run time. - The configuration file for ``libedit`` is different from that + The configuration file for ``editline`` is different from that of GNU readline. If you programmatically load configuration strings - you can check for the text "libedit" in :const:`readline.__doc__` - to differentiate between GNU readline and libedit. + you can use :data:`backend` to determine which library is being used. - If you use *editline*/``libedit`` readline emulation on macOS, the + If you use ``editline``/``libedit`` readline emulation on macOS, the initialization file located in your home directory is named ``.editrc``. For example, the following content in ``~/.editrc`` will turn ON *vi* keybindings and TAB completion:: @@ -44,6 +43,12 @@ Readline library in general. python:bind -v python:bind ^I rl_complete +.. data:: backend + + The name of the underlying Readline library being used, either + ``"readline"`` or ``"editline"``. + + .. versionadded:: 3.13 Init file --------- diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 5ebb0a7780c37b..678a11c6f45490 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`reprlib` module provides a means for producing object representations +The :mod:`!reprlib` module provides a means for producing object representations with limits on the size of the resulting strings. This is used in the Python debugger and may be useful in other contexts as well. @@ -58,29 +58,31 @@ This module provides a class, an instance, and a function: limits on most sizes. In addition to size-limiting tools, the module also provides a decorator for -detecting recursive calls to :meth:`__repr__` and substituting a placeholder -string instead. +detecting recursive calls to :meth:`~object.__repr__` and substituting a +placeholder string instead. .. index:: single: ...; placeholder .. decorator:: recursive_repr(fillvalue="...") - Decorator for :meth:`__repr__` methods to detect recursive calls within the + Decorator for :meth:`~object.__repr__` methods to detect recursive calls within the same thread. If a recursive call is made, the *fillvalue* is returned, - otherwise, the usual :meth:`__repr__` call is made. For example: - - >>> from reprlib import recursive_repr - >>> class MyList(list): - ... @recursive_repr() - ... def __repr__(self): - ... return '<' + '|'.join(map(repr, self)) + '>' - ... - >>> m = MyList('abc') - >>> m.append(m) - >>> m.append('x') - >>> print(m) - <'a'|'b'|'c'|...|'x'> + otherwise, the usual :meth:`!__repr__` call is made. For example: + + .. doctest:: + + >>> from reprlib import recursive_repr + >>> class MyList(list): + ... @recursive_repr() + ... def __repr__(self): + ... return '<' + '|'.join(map(repr, self)) + '>' + ... + >>> m = MyList('abc') + >>> m.append(m) + >>> m.append('x') + >>> print(m) + <'a'|'b'|'c'|...|'x'> .. versionadded:: 3.2 @@ -148,10 +150,10 @@ which format specific object types. with no line breaks or indentation, like the standard :func:`repr`. For example: - .. code-block:: pycon + .. doctest:: indent >>> example = [ - 1, 'spam', {'a': 2, 'b': 'spam eggs', 'c': {3: 4.5, 6: []}}, 'ham'] + ... 1, 'spam', {'a': 2, 'b': 'spam eggs', 'c': {3: 4.5, 6: []}}, 'ham'] >>> import reprlib >>> aRepr = reprlib.Repr() >>> print(aRepr.repr(example)) @@ -160,7 +162,7 @@ which format specific object types. If :attr:`~Repr.indent` is set to a string, each recursion level is placed on its own line, indented by that string: - .. code-block:: pycon + .. doctest:: indent >>> aRepr.indent = '-->' >>> print(aRepr.repr(example)) @@ -181,7 +183,7 @@ which format specific object types. Setting :attr:`~Repr.indent` to a positive integer value behaves as if it was set to a string with that number of spaces: - .. code-block:: pycon + .. doctest:: indent >>> aRepr.indent = 4 >>> print(aRepr.repr(example)) @@ -234,7 +236,9 @@ Subclassing Repr Objects The use of dynamic dispatching by :meth:`Repr.repr1` allows subclasses of :class:`Repr` to add support for additional built-in object types or to modify the handling of types already supported. This example shows how special support -for file objects could be added:: +for file objects could be added: + +.. testcode:: import reprlib import sys @@ -248,3 +252,7 @@ for file objects could be added:: aRepr = MyRepr() print(aRepr.repr(sys.stdin)) # prints '' + +.. testoutput:: + + diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index a5324c82c63484..ef65674d1b0a78 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -13,7 +13,7 @@ This module provides basic mechanisms for measuring and controlling system resources utilized by a program. -.. include:: ../includes/wasm-notavail.rst +.. availability:: Unix, not Emscripten, not WASI. Symbolic constants are used to specify particular system resources and to request usage information about either the current process or its children. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 42ed8c253b8027..406b080b7be30f 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -39,7 +39,7 @@ The :mod:`runpy` module provides two functions: The *mod_name* argument should be an absolute module name. If the module name refers to a package rather than a normal - module, then that package is imported and the ``__main__`` submodule within + module, then that package is imported and the :mod:`__main__` submodule within that package is then executed and the resulting module globals dictionary returned. @@ -74,7 +74,7 @@ The :mod:`runpy` module provides two functions: Note that this manipulation of :mod:`sys` is not thread-safe. Other threads may see the partially initialised module, as well as the altered list of - arguments. It is recommended that the :mod:`sys` module be left alone when + arguments. It is recommended that the ``sys`` module be left alone when invoking this function from threaded code. .. seealso:: @@ -82,7 +82,7 @@ The :mod:`runpy` module provides two functions: command line. .. versionchanged:: 3.1 - Added ability to execute packages by looking for a ``__main__`` submodule. + Added ability to execute packages by looking for a :mod:`__main__` submodule. .. versionchanged:: 3.2 Added ``__cached__`` global variable (see :pep:`3147`). @@ -106,15 +106,16 @@ The :mod:`runpy` module provides two functions: Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython command line, the supplied path may refer to a Python source file, a - compiled bytecode file or a valid sys.path entry containing a ``__main__`` - module (e.g. a zipfile containing a top-level ``__main__.py`` file). + compiled bytecode file or a valid :data:`sys.path` entry containing a + :mod:`__main__` module + (e.g. a zipfile containing a top-level ``__main__.py`` file). For a simple script, the specified code is simply executed in a fresh - module namespace. For a valid sys.path entry (typically a zipfile or + module namespace. For a valid :data:`sys.path` entry (typically a zipfile or directory), the entry is first added to the beginning of ``sys.path``. The function then looks for and executes a :mod:`__main__` module using the updated path. Note that there is no special protection against invoking - an existing :mod:`__main__` entry located elsewhere on ``sys.path`` if + an existing ``__main__`` entry located elsewhere on ``sys.path`` if there is no such module at the specified location. The optional dictionary argument *init_globals* may be used to pre-populate @@ -137,14 +138,14 @@ The :mod:`runpy` module provides two functions: supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If the supplied path is a reference to a valid sys.path entry, then - ``__spec__`` will be set appropriately for the imported ``__main__`` + If the supplied path is a reference to a valid :data:`sys.path` entry, then + ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be :ref:`set as normal ` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, - ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated + :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated with the value of ``path_name`` and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function @@ -152,7 +153,7 @@ The :mod:`runpy` module provides two functions: Note that, unlike :func:`run_module`, the alterations made to :mod:`sys` are not optional in this function as these adjustments are essential to - allowing the execution of sys.path entries. As the thread-safety + allowing the execution of :data:`sys.path` entries. As the thread-safety limitations still apply, use of this function in threaded code should be either serialised with the import lock or delegated to a separate process. @@ -165,7 +166,7 @@ The :mod:`runpy` module provides two functions: .. versionchanged:: 3.4 Updated to take advantage of the module spec feature added by :pep:`451`. This allows ``__cached__`` to be set correctly in the - case where ``__main__`` is imported from a valid sys.path entry rather + case where ``__main__`` is imported from a valid :data:`sys.path` entry rather than being executed directly. .. versionchanged:: 3.12 diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 04215d31ba10ca..01bac5afd0b9b3 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -115,7 +115,7 @@ Scheduler Objects .. method:: scheduler.run(blocking=True) - Run all scheduled events. This method will wait (using the :func:`delayfunc` + Run all scheduled events. This method will wait (using the *delayfunc* function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. diff --git a/Doc/library/security_warnings.rst b/Doc/library/security_warnings.rst index 284f3658320623..a573c98f73eb0a 100644 --- a/Doc/library/security_warnings.rst +++ b/Doc/library/security_warnings.rst @@ -9,7 +9,6 @@ The following modules have specific security considerations: * :mod:`base64`: :ref:`base64 security considerations ` in :rfc:`4648` -* :mod:`cgi`: :ref:`CGI security considerations ` * :mod:`hashlib`: :ref:`all constructors take a "usedforsecurity" keyword-only argument disabling known insecure and blocked algorithms ` diff --git a/Doc/library/select.rst b/Doc/library/select.rst index b0891b0c8f584a..c2941e628d9d78 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -6,10 +6,10 @@ -------------- -This module provides access to the :c:func:`select` and :c:func:`poll` functions -available in most operating systems, :c:func:`devpoll` available on -Solaris and derivatives, :c:func:`epoll` available on Linux 2.5+ and -:c:func:`kqueue` available on most BSD. +This module provides access to the :c:func:`!select` and :c:func:`!poll` functions +available in most operating systems, :c:func:`!devpoll` available on +Solaris and derivatives, :c:func:`!epoll` available on Linux 2.5+ and +:c:func:`!kqueue` available on most BSD. Note that on Windows, it only works for sockets; on other operating systems, it also works for other file types (in particular, on Unix, it works on pipes). It cannot be used on regular files to determine whether a file has grown since @@ -41,10 +41,10 @@ The module defines the following: polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. - :c:func:`devpoll` objects are linked to the number of file + :c:func:`!devpoll` objects are linked to the number of file descriptors allowed at the time of instantiation. If your program - reduces this value, :c:func:`devpoll` will fail. If your program - increases this value, :c:func:`devpoll` may return an + reduces this value, :c:func:`!devpoll` will fail. If your program + increases this value, :c:func:`!devpoll` may return an incomplete list of active file descriptors. The new file descriptor is :ref:`non-inheritable `. @@ -62,7 +62,7 @@ The module defines the following: *sizehint* informs epoll about the expected number of events to be registered. It must be positive, or ``-1`` to use the default. It is only - used on older systems where :c:func:`epoll_create1` is not available; + used on older systems where :c:func:`!epoll_create1` is not available; otherwise it has no effect (though its value is still checked). *flags* is deprecated and completely ignored. However, when supplied, its @@ -117,7 +117,7 @@ The module defines the following: .. function:: select(rlist, wlist, xlist[, timeout]) - This is a straightforward interface to the Unix :c:func:`select` system call. + This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either integers representing file descriptors or objects with a parameterless method named :meth:`~io.IOBase.fileno` returning such an integer: @@ -154,7 +154,7 @@ The module defines the following: .. index:: single: WinSock File objects on Windows are not acceptable, but sockets are. On Windows, - the underlying :c:func:`select` function is provided by the WinSock + the underlying :c:func:`!select` function is provided by the WinSock library, and does not handle file descriptors that don't originate from WinSock. @@ -169,7 +169,7 @@ The module defines the following: The minimum number of bytes which can be written without blocking to a pipe when the pipe has been reported as ready for writing by :func:`~select.select`, - :func:`poll` or another interface in this module. This doesn't apply + :func:`!poll` or another interface in this module. This doesn't apply to other kind of file-like objects such as sockets. This value is guaranteed by POSIX to be at least 512. @@ -184,11 +184,11 @@ The module defines the following: ``/dev/poll`` Polling Objects ----------------------------- -Solaris and derivatives have ``/dev/poll``. While :c:func:`select` is -O(highest file descriptor) and :c:func:`poll` is O(number of file +Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is +O(highest file descriptor) and :c:func:`!poll` is O(number of file descriptors), ``/dev/poll`` is O(active file descriptors). -``/dev/poll`` behaviour is very close to the standard :c:func:`poll` +``/dev/poll`` behaviour is very close to the standard :c:func:`!poll` object. @@ -222,7 +222,7 @@ object. implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`poll` + check for. The constants are the same that with :c:func:`!poll` object. The default value is a combination of the constants :const:`POLLIN`, :const:`POLLPRI`, and :const:`POLLOUT`. @@ -231,7 +231,7 @@ object. Registering a file descriptor that's already registered is not an error, but the result is undefined. The appropriate action is to unregister or modify it first. This is an important difference - compared with :c:func:`poll`. + compared with :c:func:`!poll`. .. method:: devpoll.modify(fd[, eventmask]) @@ -376,13 +376,13 @@ Edge and Level Trigger Polling (epoll) Objects Polling Objects --------------- -The :c:func:`poll` system call, supported on most Unix systems, provides better +The :c:func:`!poll` system call, supported on most Unix systems, provides better scalability for network servers that service many, many clients at the same -time. :c:func:`poll` scales better because the system call only requires listing -the file descriptors of interest, while :c:func:`select` builds a bitmap, turns +time. :c:func:`!poll` scales better because the system call only requires listing +the file descriptors of interest, while :c:func:`!select` builds a bitmap, turns on bits for the fds of interest, and then afterward the whole bitmap has to be -linearly scanned again. :c:func:`select` is O(highest file descriptor), while -:c:func:`poll` is O(number of file descriptors). +linearly scanned again. :c:func:`!select` is O(highest file descriptor), while +:c:func:`!poll` is O(number of file descriptors). .. method:: poll.register(fd[, eventmask]) diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index 0deb15cf4c5037..76cbf91412f763 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -21,7 +21,7 @@ It defines a :class:`BaseSelector` abstract base class, along with several concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...), that can be used to wait for I/O readiness notification on multiple file objects. In the following, "file object" refers to any object with a -:meth:`fileno()` method, or a raw file descriptor. See :term:`file object`. +:meth:`~io.IOBase.fileno` method, or a raw file descriptor. See :term:`file object`. :class:`DefaultSelector` is an alias to the most efficient implementation available on the current platform: this should be the default choice for most @@ -60,9 +60,9 @@ constants below: +-----------------------+-----------------------------------------------+ | Constant | Meaning | +=======================+===============================================+ - | :const:`EVENT_READ` | Available for read | + | .. data:: EVENT_READ | Available for read | +-----------------------+-----------------------------------------------+ - | :const:`EVENT_WRITE` | Available for write | + | .. data:: EVENT_WRITE | Available for write | +-----------------------+-----------------------------------------------+ @@ -132,8 +132,8 @@ constants below: Change a registered file object's monitored events or attached data. - This is equivalent to :meth:`BaseSelector.unregister(fileobj)` followed - by :meth:`BaseSelector.register(fileobj, events, data)`, except that it + This is equivalent to ``BaseSelector.unregister(fileobj)`` followed + by ``BaseSelector.register(fileobj, events, data)``, except that it can be implemented more efficiently. This returns a new :class:`SelectorKey` instance, or raises a diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index dc87af398ed757..88802d717d7383 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -25,7 +25,7 @@ lots of shared sub-objects. The keys are ordinary strings. database file is opened for reading and writing. The optional *flag* parameter has the same interpretation as the *flag* parameter of :func:`dbm.open`. - By default, pickles created with :data:`pickle.DEFAULT_PROTOCOL` are used + By default, pickles created with :const:`pickle.DEFAULT_PROTOCOL` are used to serialize values. The version of the pickle protocol can be specified with the *protocol* parameter. @@ -42,7 +42,7 @@ lots of shared sub-objects. The keys are ordinary strings. mutated). .. versionchanged:: 3.10 - :data:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle + :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. .. versionchanged:: 3.11 @@ -94,9 +94,9 @@ Two additional methods are supported: Restrictions ------------ - .. index:: - pair: module; dbm.ndbm - pair: module; dbm.gnu +.. index:: + pair: module; dbm.ndbm + pair: module; dbm.gnu * The choice of which database package will be used (such as :mod:`dbm.ndbm` or :mod:`dbm.gnu`) depends on which interface is available. Therefore it is not @@ -119,7 +119,7 @@ Restrictions A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. - By default, pickles created with :data:`pickle.DEFAULT_PROTOCOL` are used + By default, pickles created with :const:`pickle.DEFAULT_PROTOCOL` are used to serialize values. The version of the pickle protocol can be specified with the *protocol* parameter. See the :mod:`pickle` documentation for a discussion of the pickle protocols. @@ -143,19 +143,20 @@ Restrictions Added context manager support. .. versionchanged:: 3.10 - :data:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle + :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. .. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') - A subclass of :class:`Shelf` which exposes :meth:`first`, :meth:`!next`, - :meth:`previous`, :meth:`last` and :meth:`set_location` which are available - in the third-party :mod:`bsddb` module from `pybsddb + A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, + :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. + These are available + in the third-party :mod:`!bsddb` module from `pybsddb `_ but not in other database modules. The *dict* object passed to the constructor must support those methods. This is generally accomplished by calling one of - :func:`bsddb.hashopen`, :func:`bsddb.btopen` or :func:`bsddb.rnopen`. The + :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The optional *protocol*, *writeback*, and *keyencoding* parameters have the same interpretation as for the :class:`Shelf` class. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 0bad51833aae13..f94833ad5331a9 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -30,12 +30,6 @@ The :mod:`shlex` module defines the following functions: in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is false. - .. note:: - - Since the :func:`split` function instantiates a :class:`~shlex.shlex` - instance, passing ``None`` for *s* will read the string to split from - standard input. - .. versionchanged:: 3.12 Passing ``None`` for *s* argument now raises an exception, rather than reading :data:`sys.stdin`. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 7f408be2336824..f61ef8b0ecc7ba 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -343,6 +343,12 @@ Directory and files operations .. versionchanged:: 3.12 Added the *onexc* parameter, deprecated *onerror*. + .. versionchanged:: 3.13 + :func:`!rmtree` now ignores :exc:`FileNotFoundError` exceptions for all + but the top-level path. + Exceptions other than :exc:`OSError` and subclasses of :exc:`!OSError` + are now always propagated to the caller. + .. attribute:: rmtree.avoids_symlink_attacks Indicates whether the current platform and implementation provides a @@ -369,7 +375,7 @@ Directory and files operations If *copy_function* is given, it must be a callable that takes two arguments *src* and *dst*, and will be used to copy *src* to *dst* if :func:`os.rename` cannot be used. If the source is a directory, - :func:`copytree` is called, passing it the :func:`copy_function`. The + :func:`copytree` is called, passing it the *copy_function*. The default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. @@ -399,6 +405,12 @@ Directory and files operations total, used and free space, in bytes. *path* may be a file or a directory. + .. note:: + + On Unix filesystems, *path* must point to a path within a **mounted** + filesystem partition. On those platforms, CPython doesn't attempt to + retrieve disk usage information from non-mounted filesystems. + .. versionadded:: 3.3 .. versionchanged:: 3.8 @@ -431,7 +443,7 @@ Directory and files operations determining if the file exists and executable. When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :attr:`os.defpath`. + returning either the "PATH" value or a fallback of :data:`os.defpath`. On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the @@ -470,6 +482,12 @@ Directory and files operations or ends with an extension that is in ``PATHEXT``; and filenames that have no extension can now be found. + .. versionchanged:: 3.12.1 + On Windows, if *mode* includes ``os.X_OK``, executables with an + extension in ``PATHEXT`` will be preferred over executables without a + matching extension. + This brings behavior closer to that of Python 3.11. + .. exception:: Error This exception collects exceptions that are raised during a multi-file diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 523d1ac5001360..7ee5ece8859825 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -562,7 +562,7 @@ The :mod:`signal` module defines the following functions: Note that installing a signal handler with :func:`signal` will reset the restart behaviour to interruptible by implicitly calling - :c:func:`siginterrupt` with a true *flag* value for the given signal. + :c:func:`!siginterrupt` with a true *flag* value for the given signal. .. function:: signal(signalnum, handler) @@ -656,7 +656,7 @@ The :mod:`signal` module defines the following functions: .. function:: sigtimedwait(sigset, timeout) Like :func:`sigwaitinfo`, but takes an additional *timeout* argument - specifying a timeout. If *timeout* is specified as :const:`0`, a poll is + specifying a timeout. If *timeout* is specified as ``0``, a poll is performed. Returns :const:`None` if a timeout occurs. .. availability:: Unix. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 44f90a3b9e496f..2dc9fb09d727e2 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -19,7 +19,7 @@ Importing this module will append site-specific paths to the module search path and add a few builtins, unless :option:`-S` was used. In that case, this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific -additions, call the :func:`site.main` function. +additions, call the :func:`main` function. .. versionchanged:: 3.3 Importing the module used to trigger paths manipulation even when using @@ -109,32 +109,40 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -.. index:: pair: module; sitecustomize +:mod:`sitecustomize` +-------------------- + +.. module:: sitecustomize After these path manipulations, an attempt is made to import a module named :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages directory. If this import fails with an :exc:`ImportError` or its subclass -exception, and the exception's :attr:`name` attribute equals to ``'sitecustomize'``, +exception, and the exception's :attr:`~ImportError.name` +attribute equals to ``'sitecustomize'``, it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), attempted output from :mod:`sitecustomize` is ignored. Any other exception causes a silent and perhaps mysterious failure of the process. -.. index:: pair: module; usercustomize +:mod:`usercustomize` +-------------------- + +.. module:: usercustomize After this, an attempt is made to import a module named :mod:`usercustomize`, which can perform arbitrary user-specific customizations, if -:data:`ENABLE_USER_SITE` is true. This file is intended to be created in the +:data:`~site.ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or -its subclass exception, and the exception's :attr:`name` attribute equals to -``'usercustomize'``, it is silently ignored. +its subclass exception, and the exception's :attr:`~ImportError.name` +attribute equals to ``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of :mod:`sitecustomize` and :mod:`usercustomize` is still attempted. +.. currentmodule:: site .. _rlcompleter-config: @@ -189,9 +197,9 @@ Module contents :func:`getuserbase` hasn't been called yet. Default value is :file:`~/.local` for UNIX and macOS non-framework builds, :file:`~/Library/Python/{X.Y}` for macOS framework builds, and - :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to + :file:`{%APPDATA%}\\Python` for Windows. This value is used to compute the installation directories for scripts, data files, Python modules, - etc. for the :ref:`user installation scheme `. + etc. for the :ref:`user installation scheme `. See also :envvar:`PYTHONUSERBASE`. @@ -258,11 +266,11 @@ If it is called without arguments, it will print the contents of :data:`USER_BASE` and whether the directory exists, then the same thing for :data:`USER_SITE`, and finally the value of :data:`ENABLE_USER_SITE`. -.. cmdoption:: --user-base +.. option:: --user-base Print the path to the user base directory. -.. cmdoption:: --user-site +.. option:: --user-site Print the path to the user site-packages directory. diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 4686232b09ac47..aaec2aa1ef1dbe 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -98,21 +98,14 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. versionchanged:: 3.4 The class now supports hostname check with :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. + :const:`ssl.HAS_SNI`). .. versionchanged:: 3.9 If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket .. versionchanged:: 3.12 - The deprecated *keyfile* and *certfile* parameters have been removed. + The deprecated *keyfile* and *certfile* parameters have been removed. .. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, \ source_address=None[, timeout]) @@ -407,15 +400,8 @@ An :class:`SMTP` instance has the following methods: If there has been no previous ``EHLO`` or ``HELO`` command this session, this method tries ESMTP ``EHLO`` first. - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - .. versionchanged:: 3.12 - The deprecated *keyfile* and *certfile* parameters have been removed. + The deprecated *keyfile* and *certfile* parameters have been removed. :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. @@ -432,7 +418,7 @@ An :class:`SMTP` instance has the following methods: .. versionchanged:: 3.4 The method now supports hostname check with :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). + :const:`~ssl.HAS_SNI`). .. versionchanged:: 3.5 The error raised for lack of STARTTLS support is now the diff --git a/Doc/library/sndhdr.rst b/Doc/library/sndhdr.rst deleted file mode 100644 index fa9323e18dc348..00000000000000 --- a/Doc/library/sndhdr.rst +++ /dev/null @@ -1,104 +0,0 @@ -:mod:`sndhdr` --- Determine type of sound file -============================================== - -.. module:: sndhdr - :synopsis: Determine type of a sound file. - :deprecated: - -.. sectionauthor:: Fred L. Drake, Jr. -.. Based on comments in the module source file. - -**Source code:** :source:`Lib/sndhdr.py` - -.. index:: - single: A-LAW - single: u-LAW - -.. deprecated-removed:: 3.11 3.13 - The :mod:`sndhdr` module is deprecated - (see :pep:`PEP 594 <594#sndhdr>` for details and alternatives). - --------------- - -The :mod:`sndhdr` provides utility functions which attempt to determine the type -of sound data which is in a file. When these functions are able to determine -what type of sound data is stored in a file, they return a -:func:`~collections.namedtuple`, containing five attributes: (``filetype``, -``framerate``, ``nchannels``, ``nframes``, ``sampwidth``). The value for *type* -indicates the data type and will be one of the strings ``'aifc'``, ``'aiff'``, -``'au'``, ``'hcom'``, ``'sndr'``, ``'sndt'``, ``'voc'``, ``'wav'``, ``'8svx'``, -``'sb'``, ``'ub'``, or ``'ul'``. The *sampling_rate* will be either the actual -value or ``0`` if unknown or difficult to decode. Similarly, *channels* will be -either the number of channels or ``0`` if it cannot be determined or if the -value is difficult to decode. The value for *frames* will be either the number -of frames or ``-1``. The last item in the tuple, *bits_per_sample*, will either -be the sample size in bits or ``'A'`` for A-LAW or ``'U'`` for u-LAW. - - -.. function:: what(filename) - - Determines the type of sound data stored in the file *filename* using - :func:`whathdr`. If it succeeds, returns a namedtuple as described above, otherwise - ``None`` is returned. - - .. versionchanged:: 3.5 - Result changed from a tuple to a namedtuple. - - -.. function:: whathdr(filename) - - Determines the type of sound data stored in a file based on the file header. - The name of the file is given by *filename*. This function returns a namedtuple as - described above on success, or ``None``. - - .. versionchanged:: 3.5 - Result changed from a tuple to a namedtuple. - -The following sound header types are recognized, as listed below with the return value -from :func:`whathdr`: and :func:`what`: - -+------------+------------------------------------+ -| Value | Sound header format | -+============+====================================+ -| ``'aifc'`` | Compressed Audio Interchange Files | -+------------+------------------------------------+ -| ``'aiff'`` | Audio Interchange Files | -+------------+------------------------------------+ -| ``'au'`` | Au Files | -+------------+------------------------------------+ -| ``'hcom'`` | HCOM Files | -+------------+------------------------------------+ -| ``'sndt'`` | Sndtool Sound Files | -+------------+------------------------------------+ -| ``'voc'`` | Creative Labs Audio Files | -+------------+------------------------------------+ -| ``'wav'`` | Waveform Audio File Format Files | -+------------+------------------------------------+ -| ``'8svx'`` | 8-Bit Sampled Voice Files | -+------------+------------------------------------+ -| ``'sb'`` | Signed Byte Audio Data Files | -+------------+------------------------------------+ -| ``'ub'`` | UB Files | -+------------+------------------------------------+ -| ``'ul'`` | uLAW Audio Files | -+------------+------------------------------------+ - -.. data:: tests - - A list of functions performing the individual tests. Each function takes two - arguments: the byte-stream and an open file-like object. When :func:`what` is - called with a byte-stream, the file-like object will be ``None``. - - The test function should return a string describing the image type if the test - succeeded, or ``None`` if it failed. - -Example: - -.. code-block:: pycon - - >>> import sndhdr - >>> imghdr.what('bass.wav') - 'wav' - >>> imghdr.whathdr('bass.wav') - 'wav' - diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 13a82cf82d5908..e0a75304ef1606 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -23,7 +23,7 @@ all modern Unix systems, Windows, MacOS, and probably additional platforms. The Python interface is a straightforward transliteration of the Unix system call and library interface for sockets to Python's object-oriented style: the -:func:`.socket` function returns a :dfn:`socket object` whose methods implement +:func:`~socket.socket` function returns a :dfn:`socket object` whose methods implement the various socket system calls. Parameter types are somewhat higher-level than in the C interface: as with :meth:`read` and :meth:`write` operations on Python files, buffer allocation on receive operations is automatic, and buffer length @@ -185,7 +185,7 @@ created. Socket addresses are represented as follows: .. versionadded:: 3.7 - :const:`AF_PACKET` is a low-level interface directly to network devices. - The packets are represented by the tuple + The addresses are represented by the tuple ``(ifname, proto[, pkttype[, hatype[, addr]]])`` where: - *ifname* - String specifying the device name. @@ -193,7 +193,6 @@ created. Socket addresses are represented as follows: May be :data:`ETH_P_ALL` to capture all protocols, one of the :ref:`ETHERTYPE_* constants ` or any other Ethernet protocol number. - Value must be in network-byte-order. - *pkttype* - Optional integer specifying the packet type: - ``PACKET_HOST`` (the default) - Packet addressed to the local host. @@ -207,14 +206,14 @@ created. Socket addresses are represented as follows: - *addr* - Optional bytes-like object specifying the hardware physical address, whose interpretation depends on the device. - .. availability:: Linux >= 2.2. + .. availability:: Linux >= 2.2. - :const:`AF_QIPCRTR` is a Linux-only socket based interface for communicating with services running on co-processors in Qualcomm platforms. The address family is represented as a ``(node, port)`` tuple where the *node* and *port* are non-negative integers. - .. availability:: Linux >= 4.7. + .. availability:: Linux >= 4.7. .. versionadded:: 3.8 @@ -348,10 +347,15 @@ Constants AF_INET6 These constants represent the address (and protocol) families, used for the - first argument to :func:`.socket`. If the :const:`AF_UNIX` constant is not + first argument to :func:`~socket.socket`. If the :const:`AF_UNIX` constant is not defined then this protocol is unsupported. More constants may be available depending on the system. +.. data:: AF_UNSPEC + + :const:`AF_UNSPEC` means that + :func:`getaddrinfo` should return socket addresses for any + address family (either IPv4, IPv6, or any other) that can be used. .. data:: SOCK_STREAM SOCK_DGRAM @@ -360,7 +364,7 @@ Constants SOCK_SEQPACKET These constants represent the socket types, used for the second argument to - :func:`.socket`. More constants may be available depending on the system. + :func:`~socket.socket`. More constants may be available depending on the system. (Only :const:`SOCK_STREAM` and :const:`SOCK_DGRAM` appear to be generally useful.) @@ -380,6 +384,8 @@ Constants .. versionadded:: 3.2 +.. _socket-unix-constants: + .. data:: SO_* SOMAXCONN MSG_* @@ -397,7 +403,7 @@ Constants Many constants of these forms, documented in the Unix documentation on sockets and/or the IP protocol, are also defined in the socket module. They are - generally used in arguments to the :meth:`setsockopt` and :meth:`getsockopt` + generally used in arguments to the :meth:`~socket.setsockopt` and :meth:`~socket.getsockopt` methods of socket objects. In most cases, only those symbols that are defined in the Unix header files are defined; for a few symbols, default values are provided. @@ -436,7 +442,8 @@ Constants ``TCP_FASTOPEN_CONNECT``, ``TCP_ULP``, ``TCP_MD5SIG_EXT``, ``TCP_FASTOPEN_KEY``, ``TCP_FASTOPEN_NO_COOKIE``, ``TCP_ZEROCOPY_RECEIVE``, ``TCP_INQ``, ``TCP_TX_DELAY``. - Added ``IP_PKTINFO``. + Added ``IP_PKTINFO``, ``IP_UNBLOCK_SOURCE``, ``IP_BLOCK_SOURCE``, + ``IP_ADD_SOURCE_MEMBERSHIP``, ``IP_DROP_SOURCE_MEMBERSHIP``. .. data:: AF_CAN PF_CAN @@ -665,7 +672,7 @@ Constants HV_GUID_BROADCAST HV_GUID_CHILDREN HV_GUID_LOOPBACK - HV_GUID_LOOPBACK + HV_GUID_PARENT Constants for Windows Hyper-V sockets for host/guest communications. @@ -762,7 +769,7 @@ The following functions all create :ref:`socket objects `. Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`.socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`~socket.socket` function above. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. The newly created sockets are :ref:`non-inheritable `. @@ -857,8 +864,8 @@ The following functions all create :ref:`socket objects `. .. function:: fromfd(fd, family, type, proto=0) Duplicate the file descriptor *fd* (an integer as returned by a file object's - :meth:`fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`.socket` function + :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address + family, socket type and protocol number are as for the :func:`~socket.socket` function above. The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on @@ -923,7 +930,7 @@ The :mod:`socket` module also offers various network-related services: ``(family, type, proto, canonname, sockaddr)`` In these tuples, *family*, *type*, *proto* are all integers and are - meant to be passed to the :func:`.socket` function. *canonname* will be + meant to be passed to the :func:`~socket.socket` function. *canonname* will be a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose @@ -978,7 +985,7 @@ The :mod:`socket` module also offers various network-related services: .. function:: gethostbyname_ex(hostname) Translate a host name to IPv4 address format, extended interface. Return a - triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the host's + 3-tuple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the host's primary host name, *aliaslist* is a (possibly empty) list of alternative host names for the same address, and *ipaddrlist* is a list of IPv4 addresses for the same interface on the same host (often but not @@ -1006,7 +1013,7 @@ The :mod:`socket` module also offers various network-related services: .. function:: gethostbyaddr(ip_address) - Return a triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the + Return a 3-tuple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the primary host name responding to the given *ip_address*, *aliaslist* is a (possibly empty) list of alternative host names for the same address, and *ipaddrlist* is a list of IPv4/v6 addresses for the same interface on the same @@ -1039,7 +1046,7 @@ The :mod:`socket` module also offers various network-related services: .. function:: getprotobyname(protocolname) Translate an internet protocol name (for example, ``'icmp'``) to a constant - suitable for passing as the (optional) third argument to the :func:`.socket` + suitable for passing as the (optional) third argument to the :func:`~socket.socket` function. This is usually only needed for sockets opened in "raw" mode (:const:`SOCK_RAW`); for the normal socket modes, the correct protocol is chosen automatically if the protocol is omitted or zero. @@ -1323,7 +1330,7 @@ The :mod:`socket` module also offers various network-related services: Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket *sock*. The *fds* parameter is a sequence of file descriptors. - Consult :meth:`sendmsg` for the documentation of these parameters. + Consult :meth:`~socket.sendmsg` for the documentation of these parameters. .. availability:: Unix, Windows, not Emscripten, not WASI. @@ -1337,7 +1344,7 @@ The :mod:`socket` module also offers various network-related services: Receive up to *maxfds* file descriptors from an :const:`AF_UNIX` socket *sock*. Return ``(msg, list(fds), flags, addr)``. - Consult :meth:`recvmsg` for the documentation of these parameters. + Consult :meth:`~socket.recvmsg` for the documentation of these parameters. .. availability:: Unix, Windows, not Emscripten, not WASI. @@ -1526,7 +1533,7 @@ to sockets. Return ``True`` if socket is in blocking mode, ``False`` if in non-blocking. - This is equivalent to checking ``socket.gettimeout() == 0``. + This is equivalent to checking ``socket.gettimeout() != 0``. .. versionadded:: 3.7 @@ -2056,10 +2063,10 @@ Example Here are four minimal example programs using the TCP/IP protocol: a server that echoes all data that it receives back (servicing only one client), and a client -using it. Note that a server must perform the sequence :func:`.socket`, +using it. Note that a server must perform the sequence :func:`~socket.socket`, :meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly repeating the :meth:`~socket.accept` to service more than one client), while a -client only needs the sequence :func:`.socket`, :meth:`~socket.connect`. Also +client only needs the sequence :func:`~socket.socket`, :meth:`~socket.connect`. Also note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on the socket it is listening on but on the new socket returned by :meth:`~socket.accept`. @@ -2099,7 +2106,7 @@ The next two examples are identical to the above two, but support both IPv4 and IPv6. The server side will listen to the first address family available (it should listen to both instead). On most of IPv6-ready systems, IPv6 will take precedence and the server may not accept IPv4 traffic. The client side will try -to connect to the all addresses returned as a result of the name resolution, and +to connect to all the addresses returned as a result of the name resolution, and sends traffic to the first one connected successfully. :: # Echo server program @@ -2251,7 +2258,7 @@ This is because the previous execution has left the socket in a ``TIME_WAIT`` state, and can't be immediately reused. There is a :mod:`socket` flag to set, in order to prevent this, -:data:`socket.SO_REUSEADDR`:: +:const:`socket.SO_REUSEADDR`:: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index d65e9fe81acf8b..5fd213fa613c8d 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -116,23 +116,28 @@ server is the address family. :class:`ForkingMixIn` and the Forking classes mentioned below are only available on POSIX platforms that support :func:`~os.fork`. - :meth:`socketserver.ForkingMixIn.server_close` waits until all child - processes complete, except if - :attr:`socketserver.ForkingMixIn.block_on_close` attribute is false. + .. attribute:: block_on_close - :meth:`socketserver.ThreadingMixIn.server_close` waits until all non-daemon - threads complete, except if - :attr:`socketserver.ThreadingMixIn.block_on_close` attribute is false. Use - daemonic threads by setting - :data:`ThreadingMixIn.daemon_threads` to ``True`` to not wait until threads - complete. + :meth:`ForkingMixIn.server_close ` + waits until all child processes complete, except if + :attr:`block_on_close` attribute is ``False``. + + :meth:`ThreadingMixIn.server_close ` + waits until all non-daemon threads complete, except if + :attr:`block_on_close` attribute is ``False``. + + .. attribute:: daemon_threads + + For :class:`ThreadingMixIn` use daemonic threads by setting + :data:`ThreadingMixIn.daemon_threads ` + to ``True`` to not wait until threads complete. .. versionchanged:: 3.7 - :meth:`socketserver.ForkingMixIn.server_close` and - :meth:`socketserver.ThreadingMixIn.server_close` now waits until all + :meth:`ForkingMixIn.server_close ` and + :meth:`ThreadingMixIn.server_close ` now waits until all child processes and non-daemonic threads complete. - Add a new :attr:`socketserver.ForkingMixIn.block_on_close` class + Add a new :attr:`ForkingMixIn.block_on_close ` class attribute to opt-in for the pre-3.7 behaviour. @@ -412,13 +417,13 @@ Request Handler Objects This function must do all the work required to service a request. The default implementation does nothing. Several instance attributes are - available to it; the request is available as :attr:`self.request`; the client - address as :attr:`self.client_address`; and the server instance as - :attr:`self.server`, in case it needs access to per-server information. + available to it; the request is available as :attr:`request`; the client + address as :attr:`client_address`; and the server instance as + :attr:`server`, in case it needs access to per-server information. - The type of :attr:`self.request` is different for datagram or stream - services. For stream services, :attr:`self.request` is a socket object; for - datagram services, :attr:`self.request` is a pair of string and socket. + The type of :attr:`request` is different for datagram or stream + services. For stream services, :attr:`request` is a socket object; for + datagram services, :attr:`request` is a pair of string and socket. .. method:: finish() @@ -428,20 +433,42 @@ Request Handler Objects raises an exception, this function will not be called. + .. attribute:: request + + The *new* :class:`socket.socket` object + to be used to communicate with the client. + + + .. attribute:: client_address + + Client address returned by :meth:`BaseServer.get_request`. + + + .. attribute:: server + + :class:`BaseServer` object used for handling the request. + + .. class:: StreamRequestHandler DatagramRequestHandler These :class:`BaseRequestHandler` subclasses override the :meth:`~BaseRequestHandler.setup` and :meth:`~BaseRequestHandler.finish` - methods, and provide :attr:`self.rfile` and :attr:`self.wfile` attributes. - The :attr:`self.rfile` and :attr:`self.wfile` attributes can be - read or written, respectively, to get the request data or return data - to the client. - The :attr:`!rfile` attributes support the :class:`io.BufferedIOBase` readable interface, - and :attr:`!wfile` attributes support the :class:`!io.BufferedIOBase` writable interface. + methods, and provide :attr:`rfile` and :attr:`wfile` attributes. + + .. attribute:: rfile + + A file object from which receives the request is read. + Support the :class:`io.BufferedIOBase` readable interface. + + .. attribute:: wfile + + A file object to which the reply is written. + Support the :class:`io.BufferedIOBase` writable interface + .. versionchanged:: 3.6 - :attr:`StreamRequestHandler.wfile` also supports the + :attr:`wfile` also supports the :class:`io.BufferedIOBase` writable interface. diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst deleted file mode 100644 index d1693ea67f0ceb..00000000000000 --- a/Doc/library/spwd.rst +++ /dev/null @@ -1,82 +0,0 @@ -:mod:`spwd` --- The shadow password database -============================================ - -.. module:: spwd - :platform: Unix - :synopsis: The shadow password database (getspnam() and friends). - :deprecated: - -.. deprecated-removed:: 3.11 3.13 - The :mod:`spwd` module is deprecated - (see :pep:`PEP 594 <594#spwd>` for details and alternatives). - --------------- - -This module provides access to the Unix shadow password database. It is -available on various Unix versions. - -.. include:: ../includes/wasm-notavail.rst - -You must have enough privileges to access the shadow password database (this -usually means you have to be root). - -Shadow password database entries are reported as a tuple-like object, whose -attributes correspond to the members of the ``spwd`` structure (Attribute field -below, see ````): - -+-------+---------------+---------------------------------+ -| Index | Attribute | Meaning | -+=======+===============+=================================+ -| 0 | ``sp_namp`` | Login name | -+-------+---------------+---------------------------------+ -| 1 | ``sp_pwdp`` | Encrypted password | -+-------+---------------+---------------------------------+ -| 2 | ``sp_lstchg`` | Date of last change | -+-------+---------------+---------------------------------+ -| 3 | ``sp_min`` | Minimal number of days between | -| | | changes | -+-------+---------------+---------------------------------+ -| 4 | ``sp_max`` | Maximum number of days between | -| | | changes | -+-------+---------------+---------------------------------+ -| 5 | ``sp_warn`` | Number of days before password | -| | | expires to warn user about it | -+-------+---------------+---------------------------------+ -| 6 | ``sp_inact`` | Number of days after password | -| | | expires until account is | -| | | disabled | -+-------+---------------+---------------------------------+ -| 7 | ``sp_expire`` | Number of days since 1970-01-01 | -| | | when account expires | -+-------+---------------+---------------------------------+ -| 8 | ``sp_flag`` | Reserved | -+-------+---------------+---------------------------------+ - -The sp_namp and sp_pwdp items are strings, all others are integers. -:exc:`KeyError` is raised if the entry asked for cannot be found. - -The following functions are defined: - - -.. function:: getspnam(name) - - Return the shadow password database entry for the given user name. - - .. versionchanged:: 3.6 - Raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user - doesn't have privileges. - -.. function:: getspall() - - Return a list of all available shadow password database entries, in arbitrary - order. - - -.. seealso:: - - Module :mod:`grp` - An interface to the group database, similar to this. - - Module :mod:`pwd` - An interface to the normal password database, similar to this. - diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e7129fb3e4de6d..6dbb34a84a4c40 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -29,7 +29,7 @@ PostgreSQL or Oracle. The :mod:`!sqlite3` module was written by Gerhard Häring. It provides an SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.7.15 or newer. +requires SQLite 3.15.2 or newer. This document includes four main sections: @@ -235,11 +235,11 @@ inserted data and retrieved values from it in multiple ways. * :ref:`sqlite3-howtos` for further reading: - * :ref:`sqlite3-placeholders` - * :ref:`sqlite3-adapters` - * :ref:`sqlite3-converters` - * :ref:`sqlite3-connection-context-manager` - * :ref:`sqlite3-howto-row-factory` + * :ref:`sqlite3-placeholders` + * :ref:`sqlite3-adapters` + * :ref:`sqlite3-converters` + * :ref:`sqlite3-connection-context-manager` + * :ref:`sqlite3-howto-row-factory` * :ref:`sqlite3-explanation` for in-depth background on transaction control. @@ -266,8 +266,9 @@ Module functions :param database: The path to the database file to be opened. - Pass ``":memory:"`` to open a connection to a database that is - in RAM instead of on disk. + You can pass ``":memory:"`` to create an `SQLite database existing only + in memory `_, and open a connection + to it. :type database: :term:`path-like object` :param float timeout: @@ -298,7 +299,7 @@ Module functions Can be ``"DEFERRED"`` (default), ``"EXCLUSIVE"`` or ``"IMMEDIATE"``; or ``None`` to disable opening transactions implicitly. Has no effect unless :attr:`Connection.autocommit` is set to - :data:`~sqlite3.LEGACY_TRANSACTION_CONTROL` (the default). + :const:`~sqlite3.LEGACY_TRANSACTION_CONTROL` (the default). :type isolation_level: str | None :param bool check_same_thread: @@ -333,7 +334,7 @@ Module functions See :attr:`Connection.autocommit` and :ref:`sqlite3-transaction-control-autocommit` for more information. *autocommit* currently defaults to - :data:`~sqlite3.LEGACY_TRANSACTION_CONTROL`. + :const:`~sqlite3.LEGACY_TRANSACTION_CONTROL`. The default will change to ``False`` in a future Python release. :type autocommit: bool @@ -354,6 +355,12 @@ Module functions .. versionadded:: 3.12 The *autocommit* parameter. + .. versionchanged:: 3.13 + Positional use of the parameters *timeout*, *detect_types*, + *isolation_level*, *check_same_thread*, *factory*, *cached_statements*, + and *uri* is deprecated. + They will become keyword-only parameters in Python 3.15. + .. function:: complete_statement(statement) Return ``True`` if the string *statement* appears to contain @@ -413,15 +420,15 @@ Module functions .. function:: register_adapter(type, adapter, /) - Register an *adapter* callable to adapt the Python type *type* into an - SQLite type. + Register an *adapter* :term:`callable` to adapt the Python type *type* + into an SQLite type. The adapter is called with a Python object of type *type* as its sole argument, and must return a value of a :ref:`type that SQLite natively understands `. .. function:: register_converter(typename, converter, /) - Register the *converter* callable to convert SQLite objects of type + Register the *converter* :term:`callable` to convert SQLite objects of type *typename* into a Python object of a specific type. The converter is invoked for all SQLite values of type *typename*; it is passed a :class:`bytes` object and should return an object of the @@ -484,7 +491,7 @@ Module constants SQLITE_DENY SQLITE_IGNORE - Flags that should be returned by the *authorizer_callback* callable + Flags that should be returned by the *authorizer_callback* :term:`callable` passed to :meth:`Connection.set_authorizer`, to indicate whether: * Access is allowed (:const:`!SQLITE_OK`), @@ -522,13 +529,13 @@ Module constants the default `threading mode `_ the underlying SQLite library is compiled with. The SQLite threading modes are: - 1. **Single-thread**: In this mode, all mutexes are disabled and SQLite is - unsafe to use in more than a single thread at once. - 2. **Multi-thread**: In this mode, SQLite can be safely used by multiple - threads provided that no single database connection is used - simultaneously in two or more threads. - 3. **Serialized**: In serialized mode, SQLite can be safely used by - multiple threads with no restriction. + 1. **Single-thread**: In this mode, all mutexes are disabled and SQLite is + unsafe to use in more than a single thread at once. + 2. **Multi-thread**: In this mode, SQLite can be safely used by multiple + threads provided that no single database connection is used + simultaneously in two or more threads. + 3. **Serialized**: In serialized mode, SQLite can be safely used by + multiple threads with no restriction. The mappings from SQLite threading modes to DB-API 2.0 threadsafety levels are as follows: @@ -623,14 +630,20 @@ Connection objects * :ref:`sqlite3-connection-shortcuts` * :ref:`sqlite3-connection-context-manager` + + .. versionchanged:: 3.13 + + A :exc:`ResourceWarning` is emitted if :meth:`close` is not called before + a :class:`!Connection` object is deleted. + An SQLite database connection has the following attributes and methods: .. method:: cursor(factory=Cursor) Create and return a :class:`Cursor` object. The cursor method accepts a single optional parameter *factory*. If - supplied, this must be a callable returning an instance of :class:`Cursor` - or its subclasses. + supplied, this must be a :term:`callable` returning + an instance of :class:`Cursor` or its subclasses. .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") @@ -723,7 +736,7 @@ Connection objects If ``-1``, it may take any number of arguments. :param func: - A callable that is called when the SQL function is invoked. + A :term:`callable` that is called when the SQL function is invoked. The callable must return :ref:`a type natively supported by SQLite `. Set to ``None`` to remove an existing SQL function. @@ -734,9 +747,6 @@ Connection objects `deterministic `_, which allows SQLite to perform additional optimizations. - :raises NotSupportedError: - If *deterministic* is used with SQLite versions older than 3.8.3. - .. versionadded:: 3.8 The *deterministic* parameter. @@ -753,8 +763,13 @@ Connection objects ... print(row) ('acbd18db4cc2f85cedef654fccc4a4d8',) + .. versionchanged:: 3.13 + + Passing *name*, *narg*, and *func* as keyword arguments is deprecated. + These parameters will become positional-only in Python 3.15. + - .. method:: create_aggregate(name, /, n_arg, aggregate_class) + .. method:: create_aggregate(name, n_arg, aggregate_class) Create or remove a user-defined SQL aggregate function. @@ -807,6 +822,11 @@ Connection objects 3 + .. versionchanged:: 3.13 + + Passing *name*, *n_arg*, and *aggregate_class* as keyword arguments is deprecated. + These parameters will become positional-only in Python 3.15. + .. method:: create_window_function(name, num_params, aggregate_class, /) @@ -894,7 +914,7 @@ Connection objects [('a', 9), ('b', 12), ('c', 16), ('d', 12), ('e', 9)] - .. method:: create_collation(name, callable) + .. method:: create_collation(name, callable, /) Create a collation named *name* using the collating function *callable*. *callable* is passed two :class:`string ` arguments, @@ -948,9 +968,10 @@ Connection objects .. method:: set_authorizer(authorizer_callback) - Register callable *authorizer_callback* to be invoked for each attempt to - access a column of a table in the database. The callback should return - one of :const:`SQLITE_OK`, :const:`SQLITE_DENY`, or :const:`SQLITE_IGNORE` + Register :term:`callable` *authorizer_callback* to be invoked + for each attempt to access a column of a table in the database. + The callback should return one of :const:`SQLITE_OK`, + :const:`SQLITE_DENY`, or :const:`SQLITE_IGNORE` to signal how access to the column should be handled by the underlying SQLite library. @@ -970,10 +991,14 @@ Connection objects .. versionchanged:: 3.11 Added support for disabling the authorizer using ``None``. + .. versionchanged:: 3.13 + Passing *authorizer_callback* as a keyword argument is deprecated. + The parameter will become positional-only in Python 3.15. + .. method:: set_progress_handler(progress_handler, n) - Register callable *progress_handler* to be invoked for every *n* + Register :term:`callable` *progress_handler* to be invoked for every *n* instructions of the SQLite virtual machine. This is useful if you want to get called from SQLite during long-running operations, for example to update a GUI. @@ -985,11 +1010,15 @@ Connection objects currently executing query and cause it to raise a :exc:`DatabaseError` exception. + .. versionchanged:: 3.13 + Passing *progress_handler* as a keyword argument is deprecated. + The parameter will become positional-only in Python 3.15. + .. method:: set_trace_callback(trace_callback) - Register callable *trace_callback* to be invoked for each SQL statement - that is actually executed by the SQLite backend. + Register :term:`callable` *trace_callback* to be invoked + for each SQL statement that is actually executed by the SQLite backend. The only argument passed to the callback is the statement (as :class:`str`) that is being executed. The return value of the callback is @@ -1009,6 +1038,10 @@ Connection objects .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Passing *trace_callback* as a keyword argument is deprecated. + The parameter will become positional-only in Python 3.15. + .. method:: enable_load_extension(enabled, /) @@ -1121,6 +1154,10 @@ Connection objects f.write('%s\n' % line) con.close() + .. seealso:: + + :ref:`sqlite3-howto-encoding` + .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) @@ -1139,8 +1176,8 @@ Connection objects Defaults to ``-1``. :param progress: - If set to a callable, it is invoked with three integer arguments for - every backup iteration: + If set to a :term:`callable`, + it is invoked with three integer arguments for every backup iteration: the *status* of the last iteration, the *remaining* number of pages still to be copied, and the *total* number of pages. @@ -1187,6 +1224,10 @@ Connection objects .. versionadded:: 3.7 + .. seealso:: + + :ref:`sqlite3-howto-encoding` + .. method:: getlimit(category, /) Get a connection runtime limit. @@ -1404,43 +1445,12 @@ Connection objects .. attribute:: text_factory - A callable that accepts a :class:`bytes` parameter and returns a text - representation of it. + A :term:`callable` that accepts a :class:`bytes` parameter + and returns a text representation of it. The callable is invoked for SQLite values with the ``TEXT`` data type. By default, this attribute is set to :class:`str`. - If you want to return ``bytes`` instead, set *text_factory* to ``bytes``. - - Example: - - .. testcode:: - - con = sqlite3.connect(":memory:") - cur = con.cursor() - - AUSTRIA = "Österreich" - - # by default, rows are returned as str - cur.execute("SELECT ?", (AUSTRIA,)) - row = cur.fetchone() - assert row[0] == AUSTRIA - - # but we can make sqlite3 always return bytestrings ... - con.text_factory = bytes - cur.execute("SELECT ?", (AUSTRIA,)) - row = cur.fetchone() - assert type(row[0]) is bytes - # the bytestrings will be encoded in UTF-8, unless you stored garbage in the - # database ... - assert row[0] == AUSTRIA.encode("utf-8") - - # we can also implement a custom text_factory ... - # here we implement one that appends "foo" to all strings - con.text_factory = lambda x: x.decode("utf-8") + "foo" - cur.execute("SELECT ?", ("bar",)) - row = cur.fetchone() - assert row[0] == "barfoo" - con.close() + See :ref:`sqlite3-howto-encoding` for more details. .. attribute:: total_changes @@ -1492,7 +1502,7 @@ Cursor objects .. method:: execute(sql, parameters=(), /) - Execute SQL a single SQL statement, + Execute a single SQL statement, optionally binding Python values using :ref:`placeholders `. @@ -1545,7 +1555,7 @@ Cursor objects :raises ProgrammingError: If *sql* contains more than one SQL statement, - or is not a DML statment. + or is not a DML statement. Example: @@ -1599,7 +1609,6 @@ Cursor objects COMMIT; """) - .. method:: fetchone() If :attr:`~Cursor.row_factory` is ``None``, @@ -1819,9 +1828,9 @@ Blob objects .. method:: seek(offset, origin=os.SEEK_SET, /) Set the current access position of the blob to *offset*. The *origin* - argument defaults to :data:`os.SEEK_SET` (absolute blob positioning). - Other values for *origin* are :data:`os.SEEK_CUR` (seek relative to the - current position) and :data:`os.SEEK_END` (seek relative to the blob’s + argument defaults to :const:`os.SEEK_SET` (absolute blob positioning). + Other values for *origin* are :const:`os.SEEK_CUR` (seek relative to the + current position) and :const:`os.SEEK_END` (seek relative to the blob’s end). @@ -2404,9 +2413,9 @@ or if :attr:`~Connection.autocommit` is ``True``, the context manager does nothing. .. note:: - The context manager neither implicitly opens a new transaction - nor closes the connection. + nor closes the connection. If you need a closing context manager, consider + using :meth:`contextlib.closing`. .. testcode:: @@ -2521,6 +2530,13 @@ Queries now return :class:`!Row` objects: >>> row["RADIUS"] # Column names are case-insensitive. 6378 +.. note:: + + The ``FROM`` clause can be omitted in the ``SELECT`` statement, as in the + above example. In such cases, SQLite returns a single row with columns + defined by expressions, e.g. literals, with the given aliases + ``expr AS alias``. + You can create a custom :attr:`~Cursor.row_factory` that returns each row as a :class:`dict`, with column names mapped to values: @@ -2571,6 +2587,47 @@ With some adjustments, the above recipe can be adapted to use a instead of a :class:`~collections.namedtuple`. +.. _sqlite3-howto-encoding: + +How to handle non-UTF-8 text encodings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, :mod:`!sqlite3` uses :class:`str` to adapt SQLite values +with the ``TEXT`` data type. +This works well for UTF-8 encoded text, but it might fail for other encodings +and invalid UTF-8. +You can use a custom :attr:`~Connection.text_factory` to handle such cases. + +Because of SQLite's `flexible typing`_, it is not uncommon to encounter table +columns with the ``TEXT`` data type containing non-UTF-8 encodings, +or even arbitrary data. +To demonstrate, let's assume we have a database with ISO-8859-2 (Latin-2) +encoded text, for example a table of Czech-English dictionary entries. +Assuming we now have a :class:`Connection` instance :py:data:`!con` +connected to this database, +we can decode the Latin-2 encoded text using this :attr:`~Connection.text_factory`: + +.. testcode:: + + con.text_factory = lambda data: str(data, encoding="latin2") + +For invalid UTF-8 or arbitrary data in stored in ``TEXT`` table columns, +you can use the following technique, borrowed from the :ref:`unicode-howto`: + +.. testcode:: + + con.text_factory = lambda data: str(data, errors="surrogateescape") + +.. note:: + + The :mod:`!sqlite3` module API does not support strings + containing surrogates. + +.. seealso:: + + :ref:`unicode-howto` + + .. _sqlite3-explanation: Explanation diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 18a6c5ab4858a4..0db233e2dde33c 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -43,8 +43,10 @@ This module provides a class, :class:`ssl.SSLSocket`, which is derived from the :class:`socket.socket` type, and provides a socket-like wrapper that also encrypts and decrypts the data going over the socket with SSL. It supports additional methods such as :meth:`getpeercert`, which retrieves the -certificate of the other side of the connection, and :meth:`cipher`, which -retrieves the cipher being used for the secure connection. +certificate of the other side of the connection, :meth:`cipher`, which +retrieves the cipher being used for the secure connection or +:meth:`get_verified_chain`, :meth:`get_unverified_chain` which retrieves +certificate chain. For more sophisticated applications, the :class:`ssl.SSLContext` class helps manage settings and certificates, which can then be inherited @@ -139,7 +141,7 @@ purposes. The settings are: :data:`PROTOCOL_TLS_CLIENT` or :data:`PROTOCOL_TLS_SERVER`, :data:`OP_NO_SSLv2`, and :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and - without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH` + without unauthenticated cipher suites. Passing :const:`~Purpose.SERVER_AUTH` as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED` and either loads CA certificates (when at least one of *cafile*, *capath* or *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load @@ -320,7 +322,7 @@ Random generation Mix the given *bytes* into the SSL pseudo-random number generator. The parameter *entropy* (a float) is a lower bound on the entropy contained in - string (so you can always use :const:`0.0`). See :rfc:`1750` for more + string (so you can always use ``0.0``). See :rfc:`1750` for more information on sources of entropy. .. versionchanged:: 3.5 @@ -906,6 +908,12 @@ Constants .. versionadded:: 3.7 +.. data:: HAS_PSK + + Whether the OpenSSL library has built-in support for TLS-PSK. + + .. versionadded:: 3.13 + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list @@ -1210,6 +1218,22 @@ SSL sockets also have the following additional methods and attributes: .. versionchanged:: 3.9 IPv6 address strings no longer have a trailing new line. +.. method:: SSLSocket.get_verified_chain() + + Returns verified certificate chain provided by the other + end of the SSL channel as a list of DER-encoded bytes. + If certificate verification was disabled method acts the same as + :meth:`~SSLSocket.get_unverified_chain`. + + .. versionadded:: 3.13 + +.. method:: SSLSocket.get_unverified_chain() + + Returns raw certificate chain provided by the other + end of the SSL channel as a list of DER-encoded bytes. + + .. versionadded:: 3.13 + .. method:: SSLSocket.cipher() Returns a three-value tuple containing the name of the cipher being used, the @@ -1380,18 +1404,18 @@ to speed up repeated connections from the same clients. Here's a table showing which versions in a client (down the side) can connect to which versions in a server (along the top): - .. table:: + .. table:: - ======================== ============ ============ ============= ========= =========== =========== - *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2** - ------------------------ ------------ ------------ ------------- --------- ----------- ----------- - *SSLv2* yes no no [1]_ no no no - *SSLv3* no yes no [2]_ no no no - *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes - *TLSv1* no no yes yes no no - *TLSv1.1* no no yes no yes no - *TLSv1.2* no no yes no no yes - ======================== ============ ============ ============= ========= =========== =========== + ======================== ============ ============ ============= ========= =========== =========== + *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2** + ------------------------ ------------ ------------ ------------- --------- ----------- ----------- + *SSLv2* yes no no [1]_ no no no + *SSLv3* no yes no [2]_ no no no + *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes + *TLSv1* no no yes yes no no + *TLSv1.1* no no yes no yes no + *TLSv1.2* no no yes no no yes + ======================== ============ ============ ============= ========= =========== =========== .. rubric:: Footnotes .. [1] :class:`SSLContext` disables SSLv2 with :data:`OP_NO_SSLv2` by default. @@ -1484,9 +1508,9 @@ to speed up repeated connections from the same clients. load CA certificates from other locations, too. The *purpose* flag specifies what kind of CA certificates are loaded. The - default settings :data:`Purpose.SERVER_AUTH` loads certificates, that are + default settings :const:`Purpose.SERVER_AUTH` loads certificates, that are flagged and trusted for TLS web server authentication (client side - sockets). :data:`Purpose.CLIENT_AUTH` loads CA certificates for client + sockets). :const:`Purpose.CLIENT_AUTH` loads CA certificates for client certificate verification on the server side. .. versionadded:: 3.4 @@ -1656,8 +1680,9 @@ to speed up repeated connections from the same clients. Due to the early negotiation phase of the TLS connection, only limited methods and attributes are usable like :meth:`SSLSocket.selected_alpn_protocol` and :attr:`SSLSocket.context`. - The :meth:`SSLSocket.getpeercert`, - :meth:`SSLSocket.cipher` and :meth:`SSLSocket.compression` methods require that + The :meth:`SSLSocket.getpeercert`, :meth:`SSLSocket.get_verified_chain`, + :meth:`SSLSocket.get_unverified_chain` :meth:`SSLSocket.cipher` + and :meth:`SSLSocket.compression` methods require that the TLS connection has progressed beyond the TLS Client Hello and therefore will not return meaningful values nor can they be called safely. @@ -1729,7 +1754,7 @@ to speed up repeated connections from the same clients. Wrap an existing Python socket *sock* and return an instance of :attr:`SSLContext.sslsocket_class` (default :class:`SSLSocket`). The returned SSL socket is tied to the context, its settings and certificates. - *sock* must be a :data:`~socket.SOCK_STREAM` socket; other + *sock* must be a :const:`~socket.SOCK_STREAM` socket; other socket types are unsupported. The parameter ``server_side`` is a boolean which identifies whether @@ -1987,6 +2012,100 @@ to speed up repeated connections from the same clients. >>> ssl.create_default_context().verify_mode # doctest: +SKIP +.. method:: SSLContext.set_psk_client_callback(callback) + + Enables TLS-PSK (pre-shared key) authentication on a client-side connection. + + In general, certificate based authentication should be preferred over this method. + + The parameter ``callback`` is a callable object with the signature: + ``def callback(hint: str | None) -> tuple[str | None, bytes]``. + The ``hint`` parameter is an optional identity hint sent by the server. + The return value is a tuple in the form (client-identity, psk). + Client-identity is an optional string which may be used by the server to + select a corresponding PSK for the client. The string must be less than or + equal to ``256`` octets when UTF-8 encoded. PSK is a + :term:`bytes-like object` representing the pre-shared key. Return a zero + length PSK to reject the connection. + + Setting ``callback`` to :const:`None` removes any existing callback. + + .. note:: + When using TLS 1.3: + + - the ``hint`` parameter is always :const:`None`. + - client-identity must be a non-empty string. + + Example usage:: + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + context.maximum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers('PSK') + + # A simple lambda: + psk = bytes.fromhex('c0ffee') + context.set_psk_client_callback(lambda hint: (None, psk)) + + # A table using the hint from the server: + psk_table = { 'ServerId_1': bytes.fromhex('c0ffee'), + 'ServerId_2': bytes.fromhex('facade') + } + def callback(hint): + return 'ClientId_1', psk_table.get(hint, b'') + context.set_psk_client_callback(callback) + + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + + .. versionadded:: 3.13 + +.. method:: SSLContext.set_psk_server_callback(callback, identity_hint=None) + + Enables TLS-PSK (pre-shared key) authentication on a server-side connection. + + In general, certificate based authentication should be preferred over this method. + + The parameter ``callback`` is a callable object with the signature: + ``def callback(identity: str | None) -> bytes``. + The ``identity`` parameter is an optional identity sent by the client which can + be used to select a corresponding PSK. + The return value is a :term:`bytes-like object` representing the pre-shared key. + Return a zero length PSK to reject the connection. + + Setting ``callback`` to :const:`None` removes any existing callback. + + The parameter ``identity_hint`` is an optional identity hint string sent to + the client. The string must be less than or equal to ``256`` octets when + UTF-8 encoded. + + .. note:: + When using TLS 1.3 the ``identity_hint`` parameter is not sent to the client. + + Example usage:: + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.maximum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers('PSK') + + # A simple lambda: + psk = bytes.fromhex('c0ffee') + context.set_psk_server_callback(lambda identity: psk) + + # A table using the identity of the client: + psk_table = { 'ClientId_1': bytes.fromhex('c0ffee'), + 'ClientId_2': bytes.fromhex('facade') + } + def callback(identity): + return psk_table.get(identity, b'') + context.set_psk_server_callback(callback, 'ServerId_1') + + This method will raise :exc:`NotImplementedError` if :data:`HAS_PSK` is + ``False``. + + .. versionadded:: 3.13 + .. index:: single: certificates .. index:: single: X509 certificate @@ -2414,6 +2533,8 @@ provided. - :meth:`~SSLSocket.read` - :meth:`~SSLSocket.write` - :meth:`~SSLSocket.getpeercert` + - :meth:`~SSLSocket.get_verified_chain` + - :meth:`~SSLSocket.get_unverified_chain` - :meth:`~SSLSocket.selected_alpn_protocol` - :meth:`~SSLSocket.selected_npn_protocol` - :meth:`~SSLSocket.cipher` @@ -2592,7 +2713,7 @@ disabled by default. >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.2 and later (if +The SSL context created above will only allow TLSv1.3 and later (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 083dc5e3bcfd68..77538514598a50 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -13,8 +13,8 @@ The :mod:`stat` module defines constants and functions for interpreting the results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they -exist). For complete details about the :c:func:`stat`, :c:func:`fstat` and -:c:func:`lstat` calls, consult the documentation for your system. +exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and +:c:func:`!lstat` calls, consult the documentation for your system. .. versionchanged:: 3.4 The stat module is backed by a C implementation. @@ -89,9 +89,9 @@ mode: .. function:: S_IFMT(mode) Return the portion of the file's mode that describes the file type (used by the - :func:`S_IS\*` functions above). + :func:`!S_IS\*` functions above). -Normally, you would use the :func:`os.path.is\*` functions for testing the type +Normally, you would use the :func:`!os.path.is\*` functions for testing the type of a file; the functions here are useful when you are doing multiple tests of the same file and wish to avoid the overhead of the :c:func:`stat` system call for each test. These are also useful when checking for information about a file diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 395b324c860389..5c8ad3a7dd7380 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -14,6 +14,7 @@ .. testsetup:: * from statistics import * + import math __name__ = '' -------------- @@ -584,7 +585,7 @@ However, for reading convenience, most of the examples show sorted sequences. The *data* can be any iterable containing sample data. For meaningful results, the number of data points in *data* should be larger than *n*. - Raises :exc:`StatisticsError` if there are not at least two data points. + Raises :exc:`StatisticsError` if there is not at least one data point. The cut points are linearly interpolated from the two nearest data points. For example, if a cut point falls one-third @@ -624,6 +625,11 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.8 + .. versionchanged:: 3.13 + No longer raises an exception for an input with only a single data point. + This allows quantile estimates to be built up one sample point + at a time becoming gradually more refined with each new data point. + .. function:: covariance(x, y, /) Return the sample covariance of two inputs *x* and *y*. Covariance @@ -741,6 +747,24 @@ However, for reading convenience, most of the examples show sorted sequences. *y = slope \* x + noise* + Continuing the example from :func:`correlation`, we look to see + how well a model based on major planets can predict the orbital + distances for dwarf planets: + + .. doctest:: + + >>> model = linear_regression(period_squared, dist_cubed, proportional=True) + >>> slope = model.slope + + >>> # Dwarf planets: Pluto, Eris, Makemake, Haumea, Ceres + >>> orbital_periods = [90_560, 204_199, 111_845, 103_410, 1_680] # days + >>> predicted_dist = [math.cbrt(slope * (p * p)) for p in orbital_periods] + >>> list(map(round, predicted_dist)) + [5912, 10166, 6806, 6459, 414] + + >>> [5_906, 10_152, 6_796, 6_450, 414] # actual distance in million km + [5906, 10152, 6796, 6450, 414] + .. versionadded:: 3.10 .. versionchanged:: 3.11 @@ -828,6 +852,11 @@ of applications in statistics. number generator. This is useful for creating reproducible results, even in a multi-threading context. + .. versionchanged:: 3.13 + + Switched to a faster algorithm. To reproduce samples from previous + versions, use :func:`random.seed` and :func:`random.gauss`. + .. method:: NormalDist.pdf(x) Using a `probability density function (pdf) @@ -922,6 +951,10 @@ of applications in statistics. :class:`NormalDist` Examples and Recipes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Classic probability problems +**************************** + :class:`NormalDist` readily solves classic probability problems. For example, given `historical data for SAT exams @@ -947,6 +980,10 @@ Find the `quartiles `_ and `deciles >>> list(map(round, sat.quantiles(n=10))) [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] + +Monte Carlo inputs for simulations +********************************** + To estimate the distribution for a model than isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation `_: @@ -963,6 +1000,9 @@ Carlo simulation `_: >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP [1.4591308524824727, 1.8035946855390597, 2.175091447274739] +Approximating binomial distributions +************************************ + Normal distributions can be used to approximate `Binomial distributions `_ when the sample size is large and when the probability of a successful @@ -1000,6 +1040,10 @@ probability that the Python room will stay within its capacity limits? >>> mean(trial() <= k for i in range(10_000)) 0.8398 + +Naive bayesian classifier +************************* + Normal distributions commonly arise in machine learning problems. Wikipedia has a `nice example of a Naive Bayesian Classifier @@ -1054,6 +1098,48 @@ The final prediction goes to the largest posterior. This is known as the 'female' +Kernel density estimation +************************* + +It is possible to estimate a continuous probability density function +from a fixed number of discrete samples. + +The basic idea is to smooth the data using `a kernel function such as a +normal distribution, triangular distribution, or uniform distribution +`_. +The degree of smoothing is controlled by a single +parameter, ``h``, representing the variance of the kernel function. + +.. testcode:: + + import math + + def kde_normal(sample, h): + "Create a continuous probability density function from a sample." + # Smooth the sample with a normal distribution of variance h. + kernel_h = NormalDist(0.0, math.sqrt(h)).pdf + n = len(sample) + def pdf(x): + return sum(kernel_h(x - x_i) for x_i in sample) / n + return pdf + +`Wikipedia has an example +`_ +where we can use the ``kde_normal()`` recipe to generate and plot +a probability density function estimated from a small sample: + +.. doctest:: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde_normal(sample, h=2.25) + >>> xarr = [i/100 for i in range(-750, 1100)] + >>> yarr = [f_hat(x) for x in xarr] + +The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: + +.. image:: kde_example.png + :alt: Scatter plot of the estimated probability density function. + .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9203afbf6a4e8a..1265b5b12e492d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -44,15 +44,16 @@ Any object can be tested for truth value, for use in an :keyword:`if` or .. index:: single: true By default, an object is considered true unless its class defines either a -:meth:`__bool__` method that returns ``False`` or a :meth:`__len__` method that +:meth:`~object.__bool__` method that returns ``False`` or a +:meth:`~object.__len__` method that returns zero, when called with the object. [1]_ Here are most of the built-in objects considered false: - .. index:: - single: None (Built-in object) - single: False (Built-in object) +.. index:: + single: None (Built-in object) + single: False (Built-in object) -* constants defined to be false: ``None`` and ``False``. +* constants defined to be false: ``None`` and ``False`` * zero of any numeric type: ``0``, ``0.0``, ``0j``, ``Decimal(0)``, ``Fraction(0, 1)`` @@ -197,7 +198,7 @@ exception. Two more operations with the same syntactic priority, :keyword:`in` and :keyword:`not in`, are supported by types that are :term:`iterable` or -implement the :meth:`__contains__` method. +implement the :meth:`~object.__contains__` method. .. _typesnumeric: @@ -282,7 +283,7 @@ the operations, see :ref:`operator-summary`): +---------------------+---------------------------------+---------+--------------------+ | ``x / y`` | quotient of *x* and *y* | | | +---------------------+---------------------------------+---------+--------------------+ -| ``x // y`` | floored quotient of *x* and | \(1) | | +| ``x // y`` | floored quotient of *x* and | \(1)\(2)| | | | *y* | | | +---------------------+---------------------------------+---------+--------------------+ | ``x % y`` | remainder of ``x / y`` | \(2) | | @@ -319,8 +320,10 @@ the operations, see :ref:`operator-summary`): Notes: (1) - Also referred to as integer division. The resultant value is a whole - integer, though the result's type is not necessarily int. The result is + Also referred to as integer division. For operands of type :class:`int`, + the result has type :class:`int`. For operands of type :class:`float`, + the result has type :class:`float`. In general, the result is a whole + integer, though the result's type is not necessarily :class:`int`. The result is always rounded towards minus infinity: ``1//2`` is ``0``, ``(-1)//2`` is ``-1``, ``1//(-2)`` is ``-1``, and ``(-1)//(-2)`` is ``0``. @@ -715,7 +718,7 @@ that's defined for any rational number, and hence applies to all instances of :class:`int` and :class:`fractions.Fraction`, and all finite instances of :class:`float` and :class:`decimal.Decimal`. Essentially, this function is given by reduction modulo ``P`` for a fixed prime ``P``. The value of ``P`` is -made available to Python as the :attr:`modulus` attribute of +made available to Python as the :attr:`~sys.hash_info.modulus` attribute of :data:`sys.hash_info`. .. impl-detail:: @@ -802,6 +805,7 @@ number, :class:`float`, or :class:`complex`:: hash_value = -2 return hash_value +.. _bltin-boolean-values: .. _typebool: Boolean Type - :class:`bool` @@ -903,9 +907,9 @@ Generator Types --------------- Python's :term:`generator`\s provide a convenient way to implement the iterator -protocol. If a container object's :meth:`__iter__` method is implemented as a +protocol. If a container object's :meth:`~iterator.__iter__` method is implemented as a generator, it will automatically return an iterator object (technically, a -generator object) supplying the :meth:`__iter__` and :meth:`~generator.__next__` +generator object) supplying the :meth:`!__iter__` and :meth:`~generator.__next__` methods. More information about generators can be found in :ref:`the documentation for the yield expression `. @@ -1639,7 +1643,7 @@ expression support in the :mod:`re` module). The casefolding algorithm is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. versionadded:: 3.3 @@ -1803,7 +1807,7 @@ expression support in the :mod:`re` module). property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard - `_. + `_. .. method:: str.isascii() @@ -1939,7 +1943,7 @@ expression support in the :mod:`re` module). The lowercasing algorithm used is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. method:: str.lstrip([chars]) @@ -2014,11 +2018,14 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 -.. method:: str.replace(old, new[, count]) +.. method:: str.replace(old, new, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by - *new*. If the optional argument *count* is given, only the first *count* - occurrences are replaced. + *new*. If *count* is given, only the first *count* occurrences are replaced. + If *count* is not specified or ``-1``, then all occurrences are replaced. + + .. versionchanged:: 3.13 + *count* is now supported as a keyword argument. .. method:: str.rfind(sub[, start[, end]]) @@ -2261,7 +2268,7 @@ expression support in the :mod:`re` module). Return a copy of the string in which each character has been mapped through the given translation table. The table must be an object that implements - indexing via :meth:`__getitem__`, typically a :term:`mapping` or + indexing via :meth:`~object.__getitem__`, typically a :term:`mapping` or :term:`sequence`. When indexed by a Unicode ordinal (an integer), the table object can do any of the following: return a Unicode ordinal or a string, to map the character to one or more other characters; return @@ -2285,7 +2292,7 @@ expression support in the :mod:`re` module). The uppercasing algorithm used is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. method:: str.zfill(width) @@ -3666,7 +3673,7 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'b'`` | Bytes (any object that follows the | \(5) | | | :ref:`buffer protocol ` or has | | -| | :meth:`__bytes__`). | | +| | :meth:`~object.__bytes__`). | | +------------+-----------------------------------------------------+-------+ | ``'s'`` | ``'s'`` is an alias for ``'b'`` and should only | \(6) | | | be used for Python2/3 code bases. | | @@ -3950,7 +3957,7 @@ copying. >>> m = memoryview(bytearray(b'abc')) >>> mm = m.toreadonly() >>> mm.tolist() - [89, 98, 99] + [97, 98, 99] >>> mm[0] = 42 Traceback (most recent call last): File "", line 1, in @@ -4006,6 +4013,7 @@ copying. :mod:`struct` syntax. One of the formats must be a byte format ('B', 'b' or 'c'). The byte length of the result must be the same as the original length. + Note that all byte lengths may depend on the operating system. Cast 1D/long to 1D/unsigned bytes:: @@ -4036,8 +4044,8 @@ copying. >>> x = memoryview(b) >>> x[0] = b'a' Traceback (most recent call last): - File "", line 1, in - ValueError: memoryview: invalid value for format "B" + ... + TypeError: memoryview: invalid type for format 'B' >>> y = x.cast('c') >>> y[0] = b'a' >>> b @@ -4403,7 +4411,8 @@ The constructors for both classes work the same: :meth:`symmetric_difference_update` methods will accept any iterable as an argument. - Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and + Note, the *elem* argument to the :meth:`~object.__contains__`, + :meth:`remove`, and :meth:`discard` methods may be a set. To support searching for an equivalent frozenset, a temporary one is created from *elem*. @@ -4748,14 +4757,17 @@ support membership tests: .. versionadded:: 3.10 -Keys views are set-like since their entries are unique and :term:`hashable`. If all -values are hashable, so that ``(key, value)`` pairs are unique and hashable, -then the items view is also set-like. (Values views are not treated as set-like +Keys views are set-like since their entries are unique and :term:`hashable`. +Items views also have set-like operations since the (key, value) pairs +are unique and the keys are hashable. +If all values in an items view are hashable as well, +then the items view can interoperate with other sets. +(Values views are not treated as set-like since the entries are generally not unique.) For set-like views, all of the operations defined for the abstract base class :class:`collections.abc.Set` are available (for example, ``==``, ``<``, or ``^``). While using set operators, -set-like views accept any iterable as the other operand, unlike sets which only -accept sets as the input. +set-like views accept any iterable as the other operand, +unlike sets which only accept sets as the input. An example of dictionary view usage:: @@ -4786,10 +4798,10 @@ An example of dictionary view usage:: >>> # set operations >>> keys & {'eggs', 'bacon', 'salad'} {'bacon'} - >>> keys ^ {'sausage', 'juice'} - {'juice', 'sausage', 'bacon', 'spam'} - >>> keys | ['juice', 'juice', 'juice'] - {'juice', 'sausage', 'bacon', 'spam', 'eggs'} + >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', 'bacon', 'spam'} + True + >>> keys | ['juice', 'juice', 'juice'] == {'bacon', 'spam', 'juice'} + True >>> # get back a read-only proxy for the original dictionary >>> values.mapping @@ -4850,7 +4862,7 @@ before the statement body is executed and exited when the statement ends: The exception passed in should never be reraised explicitly - instead, this method should return a false value to indicate that the method completed successfully and does not want to suppress the raised exception. This allows - context management code to easily detect whether or not an :meth:`__exit__` + context management code to easily detect whether or not an :meth:`~object.__exit__` method has actually failed. Python defines several context managers to support easy thread synchronisation, @@ -4996,8 +5008,8 @@ exception to disallow mistakes like ``dict[str][str]``:: >>> dict[str][str] Traceback (most recent call last): - File "", line 1, in - TypeError: There are no type variables left in dict[str] + ... + TypeError: dict[str] is not a generic class However, such expressions are valid when :ref:`type variables ` are used. The index must have as many elements as there are type variable items @@ -5162,6 +5174,14 @@ enables cleaner type hinting syntax compared to :data:`typing.Union`. def square(number: int | float) -> int | float: return number ** 2 + .. note:: + + The ``|`` operand cannot be used at runtime to define unions where one or + more members is a forward reference. For example, ``int | "Foo"``, where + ``"Foo"`` is a reference to a class not yet defined, will fail at + runtime. For unions which include forward references, present the + whole expression as a string, e.g. ``"int | Foo"``. + .. describe:: union_object == other Union objects can be tested for equality with other union objects. Details: @@ -5195,13 +5215,15 @@ enables cleaner type hinting syntax compared to :data:`typing.Union`. >>> isinstance("", int | str) True - However, union objects containing :ref:`parameterized generics - ` cannot be used:: + However, :ref:`parameterized generics ` in + union objects cannot be checked:: - >>> isinstance(1, int | list[int]) + >>> isinstance(1, int | list[int]) # short-circuit evaluation + True + >>> isinstance([1], int | list[int]) Traceback (most recent call last): - File "", line 1, in - TypeError: isinstance() argument 2 cannot contain a parameterized generic + ... + TypeError: isinstance() argument 2 cannot be a parameterized generic The user-exposed type for the union object can be accessed from :data:`types.UnionType` and used for :func:`isinstance` checks. An object cannot be @@ -5216,9 +5238,11 @@ instantiated from the type:: TypeError: cannot create 'types.UnionType' instances .. note:: - The :meth:`__or__` method for type objects was added to support the syntax - ``X | Y``. If a metaclass implements :meth:`__or__`, the Union may - override it:: + The :meth:`!__or__` method for type objects was added to support the syntax + ``X | Y``. If a metaclass implements :meth:`!__or__`, the Union may + override it: + + .. doctest:: >>> class M(type): ... def __or__(self, other): @@ -5230,7 +5254,7 @@ instantiated from the type:: >>> C | int 'Hello' >>> int | C - int | __main__.C + int | C .. seealso:: @@ -5304,25 +5328,30 @@ Methods .. index:: pair: object; method Methods are functions that are called using the attribute notation. There are -two flavors: built-in methods (such as :meth:`append` on lists) and class -instance methods. Built-in methods are described with the types that support -them. +two flavors: :ref:`built-in methods ` (such as :meth:`append` +on lists) and :ref:`class instance method `. +Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) through an instance, you get a special object: a :dfn:`bound method` (also called -:dfn:`instance method`) object. When called, it will add the ``self`` argument +:ref:`instance method `) object. When called, it will add +the ``self`` argument to the argument list. Bound methods have two special read-only attributes: -``m.__self__`` is the object on which the method operates, and ``m.__func__`` is +:attr:`m.__self__ ` is the object on which the method +operates, and :attr:`m.__func__ ` is the function implementing the method. Calling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)``. -Like function objects, bound method objects support getting arbitrary +Like :ref:`function objects `, bound method objects support +getting arbitrary attributes. However, since method attributes are actually stored on the -underlying function object (``meth.__func__``), setting method attributes on +underlying function object (:attr:`method.__func__`), setting method attributes on bound methods is disallowed. Attempting to set an attribute on a method results in an :exc:`AttributeError` being raised. In order to set a method -attribute, you need to explicitly set it on the underlying function object:: +attribute, you need to explicitly set it on the underlying function object: + +.. doctest:: >>> class C: ... def method(self): @@ -5337,7 +5366,7 @@ attribute, you need to explicitly set it on the underlying function object:: >>> c.method.whoami 'my name is method' -See :ref:`types` for more information. +See :ref:`instance-methods` for more information. .. index:: object; code, code object @@ -5476,6 +5505,14 @@ types, where they are relevant. Some of these are not reported by the .. versionadded:: 3.3 +.. attribute:: definition.__type_params__ + + The :ref:`type parameters ` of generic classes, functions, + and :ref:`type aliases `. + + .. versionadded:: 3.12 + + .. attribute:: class.__mro__ This attribute is a tuple of classes that are considered when looking for @@ -5496,7 +5533,7 @@ types, where they are relevant. Some of these are not reported by the definition order. Example:: >>> int.__subclasses__() - [] + [, , , ] .. _int_max_str_digits: @@ -5532,7 +5569,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> _ = int('2' * 5432) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit >>> i = int('2' * 4300) >>> len(str(i)) 4300 @@ -5540,7 +5577,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> len(str(i_squared)) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit >>> len(hex(i_squared)) 7144 >>> assert int(hex(i_squared), base=16) == i*i # Hexadecimal is unlimited. @@ -5610,7 +5647,7 @@ From code, you can inspect the current limit and set a new one using these a getter and setter for the interpreter-wide limit. Subinterpreters have their own limit. -Information about the default and minimum can be found in :attr:`sys.int_info`: +Information about the default and minimum can be found in :data:`sys.int_info`: * :data:`sys.int_info.default_max_str_digits ` is the compiled-in default limit. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 26b3f5000634f5..262b785bbcbfc1 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -206,15 +206,15 @@ literal text, it can be escaped by doubling: ``{{`` and ``}}``. The grammar for a replacement field is as follows: - .. productionlist:: format-string - replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" - field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* - arg_name: [`identifier` | `digit`+] - attribute_name: `identifier` - element_index: `digit`+ | `index_string` - index_string: + - conversion: "r" | "s" | "a" - format_spec: +.. productionlist:: format-string + replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" + field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* + arg_name: [`identifier` | `digit`+] + attribute_name: `identifier` + element_index: `digit`+ | `index_string` + index_string: + + conversion: "r" | "s" | "a" + format_spec: In less formal terms, the replacement field can start with a *field_name* that specifies the object whose value is to be formatted and inserted @@ -227,7 +227,9 @@ See also the :ref:`formatspec` section. The *field_name* itself begins with an *arg_name* that is either a number or a keyword. If it's a number, it refers to a positional argument, and if it's a keyword, -it refers to a named keyword argument. If the numerical arg_names in a format string +it refers to a named keyword argument. An *arg_name* is treated as a number if +a call to :meth:`str.isdecimal` on the string would return true. +If the numerical arg_names in a format string are 0, 1, 2, ... in sequence, they can all be omitted (not just some) and the numbers 0, 1, 2, ... will be automatically inserted in that order. Because *arg_name* is not quote-delimited, it is not possible to specify arbitrary @@ -330,30 +332,30 @@ affect the :func:`format` function. The meaning of the various alignment options is as follows: - .. index:: - single: < (less); in string formatting - single: > (greater); in string formatting - single: = (equals); in string formatting - single: ^ (caret); in string formatting - - +---------+----------------------------------------------------------+ - | Option | Meaning | - +=========+==========================================================+ - | ``'<'`` | Forces the field to be left-aligned within the available | - | | space (this is the default for most objects). | - +---------+----------------------------------------------------------+ - | ``'>'`` | Forces the field to be right-aligned within the | - | | available space (this is the default for numbers). | - +---------+----------------------------------------------------------+ - | ``'='`` | Forces the padding to be placed after the sign (if any) | - | | but before the digits. This is used for printing fields | - | | in the form '+000000120'. This alignment option is only | - | | valid for numeric types. It becomes the default for | - | | numbers when '0' immediately precedes the field width. | - +---------+----------------------------------------------------------+ - | ``'^'`` | Forces the field to be centered within the available | - | | space. | - +---------+----------------------------------------------------------+ +.. index:: + single: < (less); in string formatting + single: > (greater); in string formatting + single: = (equals); in string formatting + single: ^ (caret); in string formatting + ++---------+----------------------------------------------------------+ +| Option | Meaning | ++=========+==========================================================+ +| ``'<'`` | Forces the field to be left-aligned within the available | +| | space (this is the default for most objects). | ++---------+----------------------------------------------------------+ +| ``'>'`` | Forces the field to be right-aligned within the | +| | available space (this is the default for numbers). | ++---------+----------------------------------------------------------+ +| ``'='`` | Forces the padding to be placed after the sign (if any) | +| | but before the digits. This is used for printing fields | +| | in the form '+000000120'. This alignment option is only | +| | valid for numeric types. It becomes the default for | +| | numbers when '0' immediately precedes the field width. | ++---------+----------------------------------------------------------+ +| ``'^'`` | Forces the field to be centered within the available | +| | space. | ++---------+----------------------------------------------------------+ Note that unless a minimum field width is defined, the field width will always be the same size as the data to fill it, so that the alignment option has no @@ -362,23 +364,23 @@ meaning in this case. The *sign* option is only valid for number types, and can be one of the following: - .. index:: - single: + (plus); in string formatting - single: - (minus); in string formatting - single: space; in string formatting - - +---------+----------------------------------------------------------+ - | Option | Meaning | - +=========+==========================================================+ - | ``'+'`` | indicates that a sign should be used for both | - | | positive as well as negative numbers. | - +---------+----------------------------------------------------------+ - | ``'-'`` | indicates that a sign should be used only for negative | - | | numbers (this is the default behavior). | - +---------+----------------------------------------------------------+ - | space | indicates that a leading space should be used on | - | | positive numbers, and a minus sign on negative numbers. | - +---------+----------------------------------------------------------+ +.. index:: + single: + (plus); in string formatting + single: - (minus); in string formatting + single: space; in string formatting + ++---------+----------------------------------------------------------+ +| Option | Meaning | ++=========+==========================================================+ +| ``'+'`` | indicates that a sign should be used for both | +| | positive as well as negative numbers. | ++---------+----------------------------------------------------------+ +| ``'-'`` | indicates that a sign should be used only for negative | +| | numbers (this is the default behavior). | ++---------+----------------------------------------------------------+ +| space | indicates that a leading space should be used on | +| | positive numbers, and a minus sign on negative numbers. | ++---------+----------------------------------------------------------+ .. index:: single: z; in string formatting diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index 5cfb533d802db4..c6d78a356d97bc 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -27,7 +27,7 @@ procedure are part of the profile. One example of a ``stringprep`` profile is ``nameprep``, which is used for internationalized domain names. The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these -tables would be very large to represent them as dictionaries or lists, the +tables would be very large to represent as dictionaries or lists, the module uses the Unicode character database internally. The module source code itself was generated using the ``mkstringprep.py`` utility. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 78fd6e397ae635..e2e6fc542e3e67 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -1,6 +1,10 @@ :mod:`struct` --- Interpret bytes as packed binary data ======================================================= +.. testsetup:: * + + from struct import * + .. module:: struct :synopsis: Interpret bytes as packed binary data. @@ -231,9 +235,9 @@ platform-dependent. | ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:expr:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:expr:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ | ``e`` | \(6) | float | 2 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ @@ -266,11 +270,11 @@ Notes: (2) When attempting to pack a non-integer using any of the integer conversion - codes, if the non-integer has a :meth:`__index__` method then that method is + codes, if the non-integer has a :meth:`~object.__index__` method then that method is called to convert the argument to an integer before packing. .. versionchanged:: 3.2 - Added use of the :meth:`__index__` method for non-integers. + Added use of the :meth:`~object.__index__` method for non-integers. (3) The ``'n'`` and ``'N'`` conversion codes are only available for the native @@ -597,6 +601,11 @@ The :mod:`struct` module also defines the following type: The calculated size of the struct (and hence of the bytes object produced by the :meth:`pack` method) corresponding to :attr:`format`. + .. versionchanged:: 3.13 The *repr()* of structs has changed. It + is now: + + >>> Struct('i') + Struct('i') .. _half precision format: https://en.wikipedia.org/wiki/Half-precision_floating-point_format diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 53dfbf827260c9..d6b892a7ed957d 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -57,10 +57,13 @@ underlying :class:`Popen` interface can be used directly. and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` instead of *capture_output*. - The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout - expires, the child process will be killed and waited for. The - :exc:`TimeoutExpired` exception will be re-raised after the child process - has terminated. + A *timeout* may be specified in seconds, it is internally passed on to + :meth:`Popen.communicate`. If the timeout expires, the child process will be + killed and waited for. The :exc:`TimeoutExpired` exception will be + re-raised after the child process has terminated. The initial process + creation itself cannot be interrupted on many platform APIs so you are not + guaranteed to see a timeout exception until at least after however long + process creation takes. The *input* argument is passed to :meth:`Popen.communicate` and thus to the subprocess's stdin. If used it must be a byte sequence, or a string if @@ -462,9 +465,9 @@ functions. :func:`open` function when creating the stdin/stdout/stderr pipe file objects: - - :const:`0` means unbuffered (read and write are one + - ``0`` means unbuffered (read and write are one system call and can return short) - - :const:`1` means line buffered + - ``1`` means line buffered (only usable if ``text=True`` or ``universal_newlines=True``) - any other positive value means use a buffer of approximately that size @@ -474,7 +477,7 @@ functions. .. versionchanged:: 3.3.1 *bufsize* now defaults to -1 to enable buffering by default to match the behavior that most code expects. In versions prior to Python 3.2.4 and - 3.3.1 it incorrectly defaulted to :const:`0` which was unbuffered + 3.3.1 it incorrectly defaulted to ``0`` which was unbuffered and allowed short reads. This was unintentional and did not match the behavior of Python 2 as most code expected. @@ -538,8 +541,8 @@ functions. :exc:`RuntimeError`. The new restriction may affect applications that are deployed in mod_wsgi, uWSGI, and other embedded environments. - If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. Otherwise + If *close_fds* is true, all file descriptors except ``0``, ``1`` and + ``2`` will be closed before the child process is executed. Otherwise when *close_fds* is false, file descriptors obey their inheritable flag as described in :ref:`fd_inheritance`. @@ -663,18 +666,18 @@ functions. passed to the underlying ``CreateProcess`` function. *creationflags*, if given, can be one or more of the following flags: - * :data:`CREATE_NEW_CONSOLE` - * :data:`CREATE_NEW_PROCESS_GROUP` - * :data:`ABOVE_NORMAL_PRIORITY_CLASS` - * :data:`BELOW_NORMAL_PRIORITY_CLASS` - * :data:`HIGH_PRIORITY_CLASS` - * :data:`IDLE_PRIORITY_CLASS` - * :data:`NORMAL_PRIORITY_CLASS` - * :data:`REALTIME_PRIORITY_CLASS` - * :data:`CREATE_NO_WINDOW` - * :data:`DETACHED_PROCESS` - * :data:`CREATE_DEFAULT_ERROR_MODE` - * :data:`CREATE_BREAKAWAY_FROM_JOB` + * :data:`CREATE_NEW_CONSOLE` + * :data:`CREATE_NEW_PROCESS_GROUP` + * :data:`ABOVE_NORMAL_PRIORITY_CLASS` + * :data:`BELOW_NORMAL_PRIORITY_CLASS` + * :data:`HIGH_PRIORITY_CLASS` + * :data:`IDLE_PRIORITY_CLASS` + * :data:`NORMAL_PRIORITY_CLASS` + * :data:`REALTIME_PRIORITY_CLASS` + * :data:`CREATE_NO_WINDOW` + * :data:`DETACHED_PROCESS` + * :data:`CREATE_DEFAULT_ERROR_MODE` + * :data:`CREATE_BREAKAWAY_FROM_JOB` *pipesize* can be used to change the size of the pipe when :data:`PIPE` is used for *stdin*, *stdout* or *stderr*. The size of the pipe @@ -734,13 +737,13 @@ arguments. code. All of the functions and methods that accept a *timeout* parameter, such as -:func:`call` and :meth:`Popen.communicate` will raise :exc:`TimeoutExpired` if +:func:`run` and :meth:`Popen.communicate` will raise :exc:`TimeoutExpired` if the timeout expires before the process exits. Exceptions defined in this module all inherit from :exc:`SubprocessError`. - .. versionadded:: 3.3 - The :exc:`SubprocessError` base class was added. +.. versionadded:: 3.3 + The :exc:`SubprocessError` base class was added. .. _subprocess-security: @@ -788,9 +791,10 @@ Instances of the :class:`Popen` class have the following methods: .. note:: - The function is implemented using a busy loop (non-blocking call and - short sleeps). Use the :mod:`asyncio` module for an asynchronous wait: - see :class:`asyncio.create_subprocess_exec`. + When the ``timeout`` parameter is not ``None``, then (on POSIX) the + function is implemented using a busy loop (non-blocking call and short + sleeps). Use the :mod:`asyncio` module for an asynchronous wait: see + :class:`asyncio.create_subprocess_exec`. .. versionchanged:: 3.3 *timeout* was added. @@ -1607,7 +1611,7 @@ improves performance. If you ever encounter a presumed highly unusual situation where you need to prevent ``vfork()`` from being used by Python, you can set the -:attr:`subprocess._USE_VFORK` attribute to a false value. +:const:`subprocess._USE_VFORK` attribute to a false value. :: @@ -1615,7 +1619,7 @@ prevent ``vfork()`` from being used by Python, you can set the Setting this has no impact on use of ``posix_spawn()`` which could use ``vfork()`` internally within its libc implementation. There is a similar -:attr:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of +:const:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of that. :: diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst deleted file mode 100644 index c7a38d96ade131..00000000000000 --- a/Doc/library/sunau.rst +++ /dev/null @@ -1,274 +0,0 @@ -:mod:`sunau` --- Read and write Sun AU files -============================================ - -.. module:: sunau - :synopsis: Provide an interface to the Sun AU sound format. - :deprecated: - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/sunau.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`sunau` module is deprecated - (see :pep:`PEP 594 <594#sunau>` for details). - --------------- - -The :mod:`sunau` module provides a convenient interface to the Sun AU sound -format. Note that this module is interface-compatible with the modules -:mod:`aifc` and :mod:`wave`. - -An audio file consists of a header followed by the data. The fields of the -header are: - -+---------------+-----------------------------------------------+ -| Field | Contents | -+===============+===============================================+ -| magic word | The four bytes ``.snd``. | -+---------------+-----------------------------------------------+ -| header size | Size of the header, including info, in bytes. | -+---------------+-----------------------------------------------+ -| data size | Physical size of the data, in bytes. | -+---------------+-----------------------------------------------+ -| encoding | Indicates how the audio samples are encoded. | -+---------------+-----------------------------------------------+ -| sample rate | The sampling rate. | -+---------------+-----------------------------------------------+ -| # of channels | The number of channels in the samples. | -+---------------+-----------------------------------------------+ -| info | ASCII string giving a description of the | -| | audio file (padded with null bytes). | -+---------------+-----------------------------------------------+ - -Apart from the info field, all header fields are 4 bytes in size. They are all -32-bit unsigned integers encoded in big-endian byte order. - -The :mod:`sunau` module defines the following functions: - - -.. function:: open(file, mode) - - If *file* is a string, open the file by that name, otherwise treat it as a - seekable file-like object. *mode* can be any of - - ``'r'`` - Read only mode. - - ``'w'`` - Write only mode. - - Note that it does not allow read/write files. - - A *mode* of ``'r'`` returns an :class:`AU_read` object, while a *mode* of ``'w'`` - or ``'wb'`` returns an :class:`AU_write` object. - - -The :mod:`sunau` module defines the following exception: - -.. exception:: Error - - An error raised when something is impossible because of Sun AU specs or - implementation deficiency. - - -The :mod:`sunau` module defines the following data items: - -.. data:: AUDIO_FILE_MAGIC - - An integer every valid Sun AU file begins with, stored in big-endian form. This - is the string ``.snd`` interpreted as an integer. - - -.. data:: AUDIO_FILE_ENCODING_MULAW_8 - AUDIO_FILE_ENCODING_LINEAR_8 - AUDIO_FILE_ENCODING_LINEAR_16 - AUDIO_FILE_ENCODING_LINEAR_24 - AUDIO_FILE_ENCODING_LINEAR_32 - AUDIO_FILE_ENCODING_ALAW_8 - - Values of the encoding field from the AU header which are supported by this - module. - - -.. data:: AUDIO_FILE_ENCODING_FLOAT - AUDIO_FILE_ENCODING_DOUBLE - AUDIO_FILE_ENCODING_ADPCM_G721 - AUDIO_FILE_ENCODING_ADPCM_G722 - AUDIO_FILE_ENCODING_ADPCM_G723_3 - AUDIO_FILE_ENCODING_ADPCM_G723_5 - - Additional known values of the encoding field from the AU header, but which are - not supported by this module. - - -.. _au-read-objects: - -AU_read Objects ---------------- - -AU_read objects, as returned by :func:`.open` above, have the following methods: - - -.. method:: AU_read.close() - - Close the stream, and make the instance unusable. (This is called automatically - on deletion.) - - -.. method:: AU_read.getnchannels() - - Returns number of audio channels (1 for mono, 2 for stereo). - - -.. method:: AU_read.getsampwidth() - - Returns sample width in bytes. - - -.. method:: AU_read.getframerate() - - Returns sampling frequency. - - -.. method:: AU_read.getnframes() - - Returns number of audio frames. - - -.. method:: AU_read.getcomptype() - - Returns compression type. Supported compression types are ``'ULAW'``, ``'ALAW'`` - and ``'NONE'``. - - -.. method:: AU_read.getcompname() - - Human-readable version of :meth:`getcomptype`. The supported types have the - respective names ``'CCITT G.711 u-law'``, ``'CCITT G.711 A-law'`` and ``'not - compressed'``. - - -.. method:: AU_read.getparams() - - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. - - -.. method:: AU_read.readframes(n) - - Reads and returns at most *n* frames of audio, as a :class:`bytes` object. The data - will be returned in linear format. If the original data is in u-LAW format, it - will be converted. - - -.. method:: AU_read.rewind() - - Rewind the file pointer to the beginning of the audio stream. - -The following two methods define a term "position" which is compatible between -them, and is otherwise implementation dependent. - - -.. method:: AU_read.setpos(pos) - - Set the file pointer to the specified position. Only values returned from - :meth:`tell` should be used for *pos*. - - -.. method:: AU_read.tell() - - Return current file pointer position. Note that the returned value has nothing - to do with the actual position in the file. - -The following two functions are defined for compatibility with the :mod:`aifc`, -and don't do anything interesting. - - -.. method:: AU_read.getmarkers() - - Returns ``None``. - - -.. method:: AU_read.getmark(id) - - Raise an error. - - -.. _au-write-objects: - -AU_write Objects ----------------- - -AU_write objects, as returned by :func:`.open` above, have the following methods: - - -.. method:: AU_write.setnchannels(n) - - Set the number of channels. - - -.. method:: AU_write.setsampwidth(n) - - Set the sample width (in bytes.) - - .. versionchanged:: 3.4 - Added support for 24-bit samples. - - -.. method:: AU_write.setframerate(n) - - Set the frame rate. - - -.. method:: AU_write.setnframes(n) - - Set the number of frames. This can be later changed, when and if more frames - are written. - - -.. method:: AU_write.setcomptype(type, name) - - Set the compression type and description. Only ``'NONE'`` and ``'ULAW'`` are - supported on output. - - -.. method:: AU_write.setparams(tuple) - - The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, with values valid for the :meth:`set\*` methods. Set all - parameters. - - -.. method:: AU_write.tell() - - Return current position in the file, with the same disclaimer for the - :meth:`AU_read.tell` and :meth:`AU_read.setpos` methods. - - -.. method:: AU_write.writeframesraw(data) - - Write audio frames, without correcting *nframes*. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: AU_write.writeframes(data) - - Write audio frames and make sure *nframes* is correct. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: AU_write.close() - - Make sure *nframes* is correct, and close the file. - - This method is called upon deletion. - -Note that it is invalid to set any parameters after calling :meth:`writeframes` -or :meth:`writeframesraw`. - diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index aaf66ea121d39c..17bfa66f043302 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -4,29 +4,12 @@ Superseded Modules ****************** -The modules described in this chapter are deprecated and only kept for +The modules described in this chapter are deprecated or :term:`soft deprecated` and only kept for backwards compatibility. They have been superseded by other modules. .. toctree:: + :maxdepth: 1 - aifc.rst - audioop.rst - cgi.rst - cgitb.rst - chunk.rst - crypt.rst - imghdr.rst - mailcap.rst - msilib.rst - nis.rst - nntplib.rst + getopt.rst optparse.rst - ossaudiodev.rst - pipes.rst - sndhdr.rst - spwd.rst - sunau.rst - telnetlib.rst - uu.rst - xdrlib.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 65ff5bfe7abd61..46159dcef940e7 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -38,7 +38,13 @@ Examining Symbol Tables .. method:: get_type() Return the type of the symbol table. Possible values are ``'class'``, - ``'module'``, and ``'function'``. + ``'module'``, ``'function'``, ``'annotation'``, ``'TypeVar bound'``, + ``'type alias'``, and ``'type parameter'``. The latter four refer to + different flavors of :ref:`annotation scopes `. + + .. versionchanged:: 3.12 + Added ``'annotation'``, ``'TypeVar bound'``, ``'type alias'``, + and ``'type parameter'`` as possible return values. .. method:: get_id() @@ -49,6 +55,10 @@ Examining Symbol Tables Return the table's name. This is the name of the class if the table is for a class, the name of the function if the table is for a function, or ``'top'`` if the table is global (:meth:`get_type` returns ``'module'``). + For type parameter scopes (which are used for generic classes, functions, + and type aliases), it is the name of the underlying class, function, or + type alias. For type alias scopes, it is the name of the type alias. + For :class:`~typing.TypeVar` bound scopes, it is the name of the ``TypeVar``. .. method:: get_lineno() @@ -197,3 +207,21 @@ Examining Symbol Tables Return the namespace bound to this name. If more than one or no namespace is bound to this name, a :exc:`ValueError` is raised. + + +.. _symtable-cli: + +Command-Line Usage +------------------ + +.. versionadded:: 3.13 + +The :mod:`symtable` module can be executed as a script from the command line. + +.. code-block:: sh + + python -m symtable [infile...] + +Symbol tables are generated for the specified Python source files and +dumped to stdout. +If no input file is specified, the content is read from stdin. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst new file mode 100644 index 00000000000000..762581b7eda7f1 --- /dev/null +++ b/Doc/library/sys.monitoring.rst @@ -0,0 +1,368 @@ +:mod:`sys.monitoring` --- Execution event monitoring +==================================================== + +.. module:: sys.monitoring + :synopsis: Access and control event monitoring + +.. versionadded:: 3.12 + +----------------- + +.. note:: + + :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, + not an independent module, so there is no need to + ``import sys.monitoring``, simply ``import sys`` and then use + ``sys.monitoring``. + + +This namespace provides access to the functions and constants necessary to +activate and control event monitoring. + +As programs execute, events occur that might be of interest to tools that +monitor execution. The :mod:`sys.monitoring` namespace provides means to +receive callbacks when events of interest occur. + +The monitoring API consists of three components: + +* `Tool identifiers`_ +* `Events`_ +* :ref:`Callbacks ` + +Tool identifiers +---------------- + +A tool identifier is an integer and the associated name. +Tool identifiers are used to discourage tools from interfering with each +other and to allow multiple tools to operate at the same time. +Currently tools are completely independent and cannot be used to +monitor each other. This restriction may be lifted in the future. + +Before registering or activating events, a tool should choose an identifier. +Identifiers are integers in the range 0 to 5 inclusive. + +Registering and using tools +''''''''''''''''''''''''''' + +.. function:: use_tool_id(tool_id: int, name: str, /) -> None + + Must be called before *tool_id* can be used. + *tool_id* must be in the range 0 to 5 inclusive. + Raises a :exc:`ValueError` if *tool_id* is in use. + +.. function:: free_tool_id(tool_id: int, /) -> None + + Should be called once a tool no longer requires *tool_id*. + +.. note:: + + :func:`free_tool_id` will not disable global or local events associated + with *tool_id*, nor will it unregister any callback functions. This + function is only intended to be used to notify the VM that the + particular *tool_id* is no longer in use. + +.. function:: get_tool(tool_id: int, /) -> str | None + + Returns the name of the tool if *tool_id* is in use, + otherwise it returns ``None``. + *tool_id* must be in the range 0 to 5 inclusive. + +All IDs are treated the same by the VM with regard to events, but the +following IDs are pre-defined to make co-operation of tools easier:: + + sys.monitoring.DEBUGGER_ID = 0 + sys.monitoring.COVERAGE_ID = 1 + sys.monitoring.PROFILER_ID = 2 + sys.monitoring.OPTIMIZER_ID = 5 + +There is no obligation to set an ID, nor is there anything preventing a tool +from using an ID even it is already in use. +However, tools are encouraged to use a unique ID and respect other tools. + +Events +------ + +The following events are supported: + +.. monitoring-event:: BRANCH + + A conditional branch is taken (or not). + +.. monitoring-event:: CALL + + A call in Python code (event occurs before the call). + +.. monitoring-event:: C_RAISE + + An exception raised from any callable, except for Python functions (event occurs after the exit). + +.. monitoring-event:: C_RETURN + + Return from any callable, except for Python functions (event occurs after the return). + +.. monitoring-event:: EXCEPTION_HANDLED + + An exception is handled. + +.. monitoring-event:: INSTRUCTION + + A VM instruction is about to be executed. + +.. monitoring-event:: JUMP + + An unconditional jump in the control flow graph is made. + +.. monitoring-event:: LINE + + An instruction is about to be executed that has a different line number from the preceding instruction. + +.. monitoring-event:: PY_RESUME + + Resumption of a Python function (for generator and coroutine functions), except for ``throw()`` calls. + +.. monitoring-event:: PY_RETURN + + Return from a Python function (occurs immediately before the return, the callee's frame will be on the stack). + +.. monitoring-event:: PY_START + + Start of a Python function (occurs immediately after the call, the callee's frame will be on the stack) + +.. monitoring-event:: PY_THROW + + A Python function is resumed by a ``throw()`` call. + +.. monitoring-event:: PY_UNWIND + + Exit from a Python function during exception unwinding. + +.. monitoring-event:: PY_YIELD + + Yield from a Python function (occurs immediately before the yield, the callee's frame will be on the stack). + +.. monitoring-event:: RAISE + + An exception is raised, except those that cause a :monitoring-event:`STOP_ITERATION` event. + +.. monitoring-event:: RERAISE + + An exception is re-raised, for example at the end of a :keyword:`finally` block. + +.. monitoring-event:: STOP_ITERATION + + An artificial :exc:`StopIteration` is raised; see `the STOP_ITERATION event`_. + + +More events may be added in the future. + +These events are attributes of the :mod:`!sys.monitoring.events` namespace. +Each event is represented as a power-of-2 integer constant. +To define a set of events, simply bitwise or the individual events together. +For example, to specify both :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_START` +events, use the expression ``PY_RETURN | PY_START``. + +.. monitoring-event:: NO_EVENTS + + An alias for ``0`` so users can do explict comparisions like:: + + if get_events(DEBUGGER_ID) == NO_EVENTS: + ... + +Events are divided into three groups: + +Local events +'''''''''''' + +Local events are associated with normal execution of the program and happen +at clearly defined locations. All local events can be disabled. +The local events are: + +* :monitoring-event:`PY_START` +* :monitoring-event:`PY_RESUME` +* :monitoring-event:`PY_RETURN` +* :monitoring-event:`PY_YIELD` +* :monitoring-event:`CALL` +* :monitoring-event:`LINE` +* :monitoring-event:`INSTRUCTION` +* :monitoring-event:`JUMP` +* :monitoring-event:`BRANCH` +* :monitoring-event:`STOP_ITERATION` + +Ancillary events +'''''''''''''''' + +Ancillary events can be monitored like other events, but are controlled +by another event: + +* :monitoring-event:`C_RAISE` +* :monitoring-event:`C_RETURN` + +The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events +are controlled by the :monitoring-event:`CALL` event. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the +corresponding :monitoring-event:`CALL` event is being monitored. + +Other events +'''''''''''' + +Other events are not necessarily tied to a specific location in the +program and cannot be individually disabled. + +The other events that can be monitored are: + +* :monitoring-event:`PY_THROW` +* :monitoring-event:`PY_UNWIND` +* :monitoring-event:`RAISE` +* :monitoring-event:`EXCEPTION_HANDLED` + + +The STOP_ITERATION event +'''''''''''''''''''''''' + +:pep:`PEP 380 <380#use-of-stopiteration-to-return-values>` +specifies that a :exc:`StopIteration` exception is raised when returning a value +from a generator or coroutine. However, this is a very inefficient way to +return a value, so some Python implementations, notably CPython 3.12+, do not +raise an exception unless it would be visible to other code. + +To allow tools to monitor for real exceptions without slowing down generators +and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. +:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. + + +Turning events on and off +------------------------- + +In order to monitor an event, it must be turned on and a corresponding callback +must be registered. +Events can be turned on or off by setting the events either globally or +for a particular code object. + + +Setting events globally +''''''''''''''''''''''' + +Events can be controlled globally by modifying the set of events being monitored. + +.. function:: get_events(tool_id: int, /) -> int + + Returns the ``int`` representing all the active events. + +.. function:: set_events(tool_id: int, event_set: int, /) -> None + + Activates all events which are set in *event_set*. + Raises a :exc:`ValueError` if *tool_id* is not in use. + +No events are active by default. + +Per code object events +'''''''''''''''''''''' + +Events can also be controlled on a per code object basis. + +.. function:: get_local_events(tool_id: int, code: CodeType, /) -> int + + Returns all the local events for *code* + +.. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None + + Activates all the local events for *code* which are set in *event_set*. + Raises a :exc:`ValueError` if *tool_id* is not in use. + +Local events add to global events, but do not mask them. +In other words, all global events will trigger for a code object, +regardless of the local events. + + +Disabling events +'''''''''''''''' + +.. data:: DISABLE + + A special value that can be returned from a callback function to disable + events for the current code location. + +Local events can be disabled for a specific code location by returning +:data:`sys.monitoring.DISABLE` from a callback function. This does not change +which events are set, or any other code locations for the same event. + +Disabling events for specific locations is very important for high +performance monitoring. For example, a program can be run under a +debugger with no overhead if the debugger disables all monitoring +except for a few breakpoints. + +.. function:: restart_events() -> None + + Enable all the events that were disabled by :data:`sys.monitoring.DISABLE` + for all tools. + + +.. _callbacks: + +Registering callback functions +------------------------------ + +To register a callable for events call + +.. function:: register_callback(tool_id: int, event: int, func: Callable | None, /) -> Callable | None + + Registers the callable *func* for the *event* with the given *tool_id* + + If another callback was registered for the given *tool_id* and *event*, + it is unregistered and returned. + Otherwise :func:`register_callback` returns ``None``. + + +Functions can be unregistered by calling +``sys.monitoring.register_callback(tool_id, event, None)``. + +Callback functions can be registered and unregistered at any time. + +Registering or unregistering a callback function will generate a :func:`sys.audit` event. + + +Callback function arguments +''''''''''''''''''''''''''' + +.. data:: MISSING + + A special value that is passed to a callback function to indicate + that there are no arguments to the call. + +When an active event occurs, the registered callback function is called. +Different events will provide the callback function with different arguments, as follows: + +* :monitoring-event:`PY_START` and :monitoring-event:`PY_RESUME`:: + + func(code: CodeType, instruction_offset: int) -> DISABLE | Any + +* :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_YIELD`:: + + func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any + +* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN`:: + + func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any + + If there are no arguments, *arg0* is set to :data:`sys.monitoring.MISSING`. + +* :monitoring-event:`RAISE`, :monitoring-event:`RERAISE`, :monitoring-event:`EXCEPTION_HANDLED`, + :monitoring-event:`PY_UNWIND`, :monitoring-event:`PY_THROW` and :monitoring-event:`STOP_ITERATION`:: + + func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any + +* :monitoring-event:`LINE`:: + + func(code: CodeType, line_number: int) -> DISABLE | Any + +* :monitoring-event:`BRANCH` and :monitoring-event:`JUMP`:: + + func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any + + Note that the *destination_offset* is where the code will next execute. + For an untaken branch this will be the offset of the instruction following + the branch. + +* :monitoring-event:`INSTRUCTION`:: + + func(code: CodeType, instruction_offset: int) -> DISABLE | Any diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index bacf8ceac5041e..aaf79205d44282 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -22,6 +22,8 @@ always available. .. versionadded:: 3.2 + .. availability:: Unix. + .. function:: addaudithook(hook) @@ -166,14 +168,18 @@ always available. Python interpreter. (This information is not available in any other way --- ``modules.keys()`` only lists the imported modules.) - See also the :attr:`sys.stdlib_module_names` list. + See also the :data:`sys.stdlib_module_names` list. .. function:: call_tracing(func, args) Call ``func(*args)``, while tracing is enabled. The tracing state is saved, and restored afterwards. This is intended to be called from a debugger from - a checkpoint, to recursively debug some other code. + a checkpoint, to recursively debug or profile some other code. + + Tracing is suspended while calling a tracing function set by + :func:`settrace` or :func:`setprofile` to avoid infinite recursion. + :func:`!call_tracing` enables explicit recursion of the tracing function. .. data:: copyright @@ -333,23 +339,21 @@ always available. *wasm32-emscripten* platform. The named tuple is provisional and may change in the future. - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------------+ - | Attribute | Explanation | - +=============================+==============================================+ - | :const:`emscripten_version` | Emscripten version as tuple of ints | - | | (major, minor, micro), e.g. ``(3, 1, 8)``. | - +-----------------------------+----------------------------------------------+ - | :const:`runtime` | Runtime string, e.g. browser user agent, | - | | ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. | - +-----------------------------+----------------------------------------------+ - | :const:`pthreads` | ``True`` if Python is compiled with | - | | Emscripten pthreads support. | - +-----------------------------+----------------------------------------------+ - | :const:`shared_memory` | ``True`` if Python is compiled with shared | - | | memory support. | - +-----------------------------+----------------------------------------------+ + .. attribute:: _emscripten_info.emscripten_version + + Emscripten version as tuple of ints (major, minor, micro), e.g. ``(3, 1, 8)``. + + .. attribute:: _emscripten_info.runtime + + Runtime string, e.g. browser user agent, ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. + + .. attribute:: _emscripten_info.pthreads + + ``True`` if Python is compiled with Emscripten pthreads support. + + .. attribute:: _emscripten_info.shared_memory + + ``True`` if Python is compiled with shared memory support. .. availability:: Emscripten. @@ -380,7 +384,7 @@ always available. This function prints out a given traceback and exception to ``sys.stderr``. - When an exception is raised and uncaught, the interpreter calls + When an exception other than :exc:`SystemExit` is raised and uncaught, the interpreter calls ``sys.excepthook`` with three arguments, the exception class, exception instance, and a traceback object. In an interactive session this happens just before control is returned to the prompt; in a Python program this happens just @@ -515,27 +519,62 @@ always available. The :term:`named tuple` *flags* exposes the status of command line flags. The attributes are read only. - ============================= ============================================================================================================== - attribute flag - ============================= ============================================================================================================== - :const:`debug` :option:`-d` - :const:`inspect` :option:`-i` - :const:`interactive` :option:`-i` - :const:`isolated` :option:`-I` - :const:`optimize` :option:`-O` or :option:`-OO` - :const:`dont_write_bytecode` :option:`-B` - :const:`no_user_site` :option:`-s` - :const:`no_site` :option:`-S` - :const:`ignore_environment` :option:`-E` - :const:`verbose` :option:`-v` - :const:`bytes_warning` :option:`-b` - :const:`quiet` :option:`-q` - :const:`hash_randomization` :option:`-R` - :const:`dev_mode` :option:`-X dev <-X>` (:ref:`Python Development Mode `) - :const:`utf8_mode` :option:`-X utf8 <-X>` - :const:`safe_path` :option:`-P` - :const:`int_max_str_digits` :option:`-X int_max_str_digits <-X>` (:ref:`integer string conversion length limitation `) - ============================= ============================================================================================================== + .. list-table:: + + * - .. attribute:: flags.debug + - :option:`-d` + + * - .. attribute:: flags.inspect + - :option:`-i` + + * - .. attribute:: flags.interactive + - :option:`-i` + + * - .. attribute:: flags.isolated + - :option:`-I` + + * - .. attribute:: flags.optimize + - :option:`-O` or :option:`-OO` + + * - .. attribute:: flags.dont_write_bytecode + - :option:`-B` + + * - .. attribute:: flags.no_user_site + - :option:`-s` + + * - .. attribute:: flags.no_site + - :option:`-S` + + * - .. attribute:: flags.ignore_environment + - :option:`-E` + + * - .. attribute:: flags.verbose + - :option:`-v` + + * - .. attribute:: flags.bytes_warning + - :option:`-b` + + * - .. attribute:: flags.quiet + - :option:`-q` + + * - .. attribute:: flags.hash_randomization + - :option:`-R` + + * - .. attribute:: flags.dev_mode + - :option:`-X dev <-X>` (:ref:`Python Development Mode `) + + * - .. attribute:: flags.utf8_mode + - :option:`-X utf8 <-X>` + + * - .. attribute:: flags.safe_path + - :option:`-P` + + * - .. attribute:: flags.int_max_str_digits + - :option:`-X int_max_str_digits <-X>` + (:ref:`integer string conversion length limitation `) + + * - .. attribute:: flags.warn_default_encoding + - :option:`-X warn_default_encoding <-X>` .. versionchanged:: 3.2 Added ``quiet`` attribute for the new :option:`-q` flag. @@ -554,6 +593,9 @@ always available. Mode ` and the ``utf8_mode`` attribute for the new :option:`-X` ``utf8`` flag. + .. versionchanged:: 3.10 + Added ``warn_default_encoding`` attribute for :option:`-X` ``warn_default_encoding`` flag. + .. versionchanged:: 3.11 Added the ``safe_path`` attribute for :option:`-P` option. @@ -570,61 +612,82 @@ always available. programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard [C99]_, 'Characteristics of floating types', for details. - .. tabularcolumns:: |l|l|L| - - +---------------------+---------------------+--------------------------------------------------+ - | attribute | float.h macro | explanation | - +=====================+=====================+==================================================+ - | ``epsilon`` | ``DBL_EPSILON`` | difference between 1.0 and the least value | - | | | greater than 1.0 that is representable as a float| - | | | | - | | | See also :func:`math.ulp`. | - +---------------------+---------------------+--------------------------------------------------+ - | ``dig`` | ``DBL_DIG`` | maximum number of decimal digits that can be | - | | | faithfully represented in a float; see below | - +---------------------+---------------------+--------------------------------------------------+ - | ``mant_dig`` | ``DBL_MANT_DIG`` | float precision: the number of base-``radix`` | - | | | digits in the significand of a float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max`` | ``DBL_MAX`` | maximum representable positive finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_exp`` | ``DBL_MAX_EXP`` | maximum integer *e* such that ``radix**(e-1)`` is| - | | | a representable finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_10_exp`` | ``DBL_MAX_10_EXP`` | maximum integer *e* such that ``10**e`` is in the| - | | | range of representable finite floats | - +---------------------+---------------------+--------------------------------------------------+ - | ``min`` | ``DBL_MIN`` | minimum representable positive *normalized* float| - | | | | - | | | Use :func:`math.ulp(0.0) ` to get the | - | | | smallest positive *denormalized* representable | - | | | float. | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_exp`` | ``DBL_MIN_EXP`` | minimum integer *e* such that ``radix**(e-1)`` is| - | | | a normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_10_exp`` | ``DBL_MIN_10_EXP`` | minimum integer *e* such that ``10**e`` is a | - | | | normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``radix`` | ``FLT_RADIX`` | radix of exponent representation | - +---------------------+---------------------+--------------------------------------------------+ - | ``rounds`` | ``FLT_ROUNDS`` | integer representing the rounding mode for | - | | | floating-point arithmetic. This reflects the | - | | | value of the system ``FLT_ROUNDS`` macro at | - | | | interpreter startup time: | - | | | ``-1`` indeterminable, | - | | | ``0`` toward zero, | - | | | ``1`` to nearest, | - | | | ``2`` toward positive infinity, | - | | | ``3`` toward negative infinity | - | | | | - | | | All other values for ``FLT_ROUNDS`` characterize | - | | | implementation-defined rounding behavior. | - +---------------------+---------------------+--------------------------------------------------+ + .. list-table:: Attributes of the :data:`!float_info` :term:`named tuple` + :header-rows: 1 + + * - attribute + - float.h macro + - explanation + + * - .. attribute:: float_info.epsilon + - :c:macro:`!DBL_EPSILON` + - difference between 1.0 and the least value greater than 1.0 that is + representable as a float. + + See also :func:`math.ulp`. + + * - .. attribute:: float_info.dig + - :c:macro:`!DBL_DIG` + - The maximum number of decimal digits that can be faithfully + represented in a float; see below. + + * - .. attribute:: float_info.mant_dig + - :c:macro:`!DBL_MANT_DIG` + - Float precision: the number of base-``radix`` digits in the + significand of a float. + + * - .. attribute:: float_info.max + - :c:macro:`!DBL_MAX` + - The maximum representable positive finite float. + + * - .. attribute:: float_info.max_exp + - :c:macro:`!DBL_MAX_EXP` + - The maximum integer *e* such that ``radix**(e-1)`` is a representable + finite float. + + * - .. attribute:: float_info.max_10_exp + - :c:macro:`!DBL_MAX_10_EXP` + - The maximum integer *e* such that ``10**e`` is in the range of + representable finite floats. + + * - .. attribute:: float_info.min + - :c:macro:`!DBL_MIN` + - The minimum representable positive *normalized* float. + + Use :func:`math.ulp(0.0) ` to get the smallest positive + *denormalized* representable float. + + * - .. attribute:: float_info.min_exp + - :c:macro:`!DBL_MIN_EXP` + - The minimum integer *e* such that ``radix**(e-1)`` is a normalized + float. + + * - .. attribute:: float_info.min_10_exp + - :c:macro:`!DBL_MIN_10_EXP` + - The minimum integer *e* such that ``10**e`` is a normalized float. + + * - .. attribute:: float_info.radix + - :c:macro:`!FLT_RADIX` + - The radix of exponent representation. + + * - .. attribute:: float_info.rounds + - :c:macro:`!FLT_ROUNDS` + - An integer representing the rounding mode for floating-point arithmetic. + This reflects the value of the system :c:macro:`!FLT_ROUNDS` macro + at interpreter startup time: + + * ``-1``: indeterminable + * ``0``: toward zero + * ``1``: to nearest + * ``2``: toward positive infinity + * ``3``: toward negative infinity + + All other values for :c:macro:`!FLT_ROUNDS` characterize + implementation-defined rounding behavior. The attribute :attr:`sys.float_info.dig` needs further explanation. If ``s`` is any string representing a decimal number with at most - :attr:`sys.float_info.dig` significant digits, then converting ``s`` to a + :attr:`!sys.float_info.dig` significant digits, then converting ``s`` to a float and back again will recover a string representing the same decimal value:: @@ -696,8 +759,8 @@ always available. Return the current value of the flags that are used for :c:func:`dlopen` calls. Symbolic names for the flag values can be - found in the :mod:`os` module (``RTLD_xxx`` constants, e.g. - :data:`os.RTLD_LAZY`). + found in the :mod:`os` module (:samp:`RTLD_{xxx}` constants, e.g. + :const:`os.RTLD_LAZY`). .. availability:: Unix. @@ -766,6 +829,15 @@ always available. higher than you might expect, because it includes the (temporary) reference as an argument to :func:`getrefcount`. + Note that the returned value may not actually reflect how many + references to the object are actually held. For example, some + objects are :term:`immortal` and have a very high refcount that does not + reflect the actual number of references. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. + + .. versionchanged:: 3.12 + Immortal objects have very large refcounts that do not match + the actual number of references to the object. .. function:: getrecursionlimit() @@ -873,24 +945,24 @@ always available. ``sys.getwindowsversion().major``. For compatibility with prior versions, only the first 5 elements are retrievable by indexing. - *platform* will be :const:`2 (VER_PLATFORM_WIN32_NT)`. + *platform* will be ``2`` (VER_PLATFORM_WIN32_NT). *product_type* may be one of the following values: +---------------------------------------+---------------------------------+ | Constant | Meaning | +=======================================+=================================+ - | :const:`1 (VER_NT_WORKSTATION)` | The system is a workstation. | + | ``1`` (VER_NT_WORKSTATION) | The system is a workstation. | +---------------------------------------+---------------------------------+ - | :const:`2 (VER_NT_DOMAIN_CONTROLLER)` | The system is a domain | + | ``2`` (VER_NT_DOMAIN_CONTROLLER) | The system is a domain | | | controller. | +---------------------------------------+---------------------------------+ - | :const:`3 (VER_NT_SERVER)` | The system is a server, but not | + | ``3`` (VER_NT_SERVER) | The system is a server, but not | | | a domain controller. | +---------------------------------------+---------------------------------+ - This function wraps the Win32 :c:func:`GetVersionEx` function; see the - Microsoft documentation on :c:func:`OSVERSIONINFOEX` for more information + This function wraps the Win32 :c:func:`!GetVersionEx` function; see the + Microsoft documentation on :c:func:`!OSVERSIONINFOEX` for more information about these fields. *platform_version* returns the major version, minor version and @@ -948,28 +1020,37 @@ always available. implementation. For more details about hashing of numeric types, see :ref:`numeric-hash`. - +---------------------+--------------------------------------------------+ - | attribute | explanation | - +=====================+==================================================+ - | :const:`width` | width in bits used for hash values | - +---------------------+--------------------------------------------------+ - | :const:`modulus` | prime modulus P used for numeric hash scheme | - +---------------------+--------------------------------------------------+ - | :const:`inf` | hash value returned for a positive infinity | - +---------------------+--------------------------------------------------+ - | :const:`nan` | (this attribute is no longer used) | - +---------------------+--------------------------------------------------+ - | :const:`imag` | multiplier used for the imaginary part of a | - | | complex number | - +---------------------+--------------------------------------------------+ - | :const:`algorithm` | name of the algorithm for hashing of str, bytes, | - | | and memoryview | - +---------------------+--------------------------------------------------+ - | :const:`hash_bits` | internal output size of the hash algorithm | - +---------------------+--------------------------------------------------+ - | :const:`seed_bits` | size of the seed key of the hash algorithm | - +---------------------+--------------------------------------------------+ + .. attribute:: hash_info.width + + The width in bits used for hash values + + .. attribute:: hash_info.modulus + The prime modulus P used for numeric hash scheme + + .. attribute:: hash_info.inf + + The hash value returned for a positive infinity + + .. attribute:: hash_info.nan + + (This attribute is no longer used) + + .. attribute:: hash_info.imag + + The multiplier used for the imaginary part of a complex number + + .. attribute:: hash_info.algorithm + + The name of the algorithm for hashing of str, bytes, and memoryview + + .. attribute:: hash_info.hash_bits + + The internal output size of the hash algorithm + + .. attribute:: hash_info.seed_bits + + The size of the seed key of the hash algorithm .. versionadded:: 3.2 @@ -1047,32 +1128,31 @@ always available. A :term:`named tuple` that holds information about Python's internal representation of integers. The attributes are read only. - .. tabularcolumns:: |l|L| - - +----------------------------------------+-----------------------------------------------+ - | Attribute | Explanation | - +========================================+===============================================+ - | :const:`bits_per_digit` | number of bits held in each digit. Python | - | | integers are stored internally in base | - | | ``2**int_info.bits_per_digit`` | - +----------------------------------------+-----------------------------------------------+ - | :const:`sizeof_digit` | size in bytes of the C type used to | - | | represent a digit | - +----------------------------------------+-----------------------------------------------+ - | :const:`default_max_str_digits` | default value for | - | | :func:`sys.get_int_max_str_digits` when it | - | | is not otherwise explicitly configured. | - +----------------------------------------+-----------------------------------------------+ - | :const:`str_digits_check_threshold` | minimum non-zero value for | - | | :func:`sys.set_int_max_str_digits`, | - | | :envvar:`PYTHONINTMAXSTRDIGITS`, or | - | | :option:`-X int_max_str_digits <-X>`. | - +----------------------------------------+-----------------------------------------------+ + .. attribute:: int_info.bits_per_digit + + The number of bits held in each digit. + Python integers are stored internally in base ``2**int_info.bits_per_digit``. + + .. attribute:: int_info.sizeof_digit + + The size in bytes of the C type used to represent a digit. + + .. attribute:: int_info.default_max_str_digits + + The default value for :func:`sys.get_int_max_str_digits` + when it is not otherwise explicitly configured. + + .. attribute:: int_info.str_digits_check_threshold + + The minimum non-zero value for :func:`sys.set_int_max_str_digits`, + :envvar:`PYTHONINTMAXSTRDIGITS`, or :option:`-X int_max_str_digits <-X>`. .. versionadded:: 3.1 .. versionchanged:: 3.11 - Added ``default_max_str_digits`` and ``str_digits_check_threshold``. + + Added :attr:`~int_info.default_max_str_digits` and + :attr:`~int_info.str_digits_check_threshold`. .. data:: __interactivehook__ @@ -1102,14 +1182,14 @@ always available. names used in Python programs are automatically interned, and the dictionaries used to hold module, class or instance attributes have interned keys. - Interned strings are not immortal; you must keep a reference to the return - value of :func:`intern` around to benefit from it. + Interned strings are not :term:`immortal`; you must keep a reference to the + return value of :func:`intern` around to benefit from it. .. function:: is_finalizing() - Return :const:`True` if the Python interpreter is - :term:`shutting down `, :const:`False` otherwise. + Return :const:`True` if the main Python interpreter is + :term:`shutting down `. Return :const:`False` otherwise. .. versionadded:: 3.5 @@ -1125,6 +1205,18 @@ always available. .. versionadded:: 3.12 +.. function:: _is_interned(string) + + Return :const:`True` if the given string is "interned", :const:`False` + otherwise. + + .. versionadded:: 3.13 + + .. impl-detail:: + + It is not guaranteed to exist in all implementations of Python. + + .. data:: last_type last_value last_traceback @@ -1287,20 +1379,20 @@ always available. ================ =========================== .. versionchanged:: 3.3 - On Linux, :attr:`sys.platform` doesn't contain the major version anymore. + On Linux, :data:`sys.platform` doesn't contain the major version anymore. It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since older Python versions include the version number, it is recommended to always use the ``startswith`` idiom presented above. .. versionchanged:: 3.8 - On AIX, :attr:`sys.platform` doesn't contain the major version anymore. + On AIX, :data:`sys.platform` doesn't contain the major version anymore. It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. Since older Python versions include the version number, it is recommended to always use the ``startswith`` idiom presented above. .. seealso:: - :attr:`os.name` has a coarser granularity. :func:`os.uname` gives + :data:`os.name` has a coarser granularity. :func:`os.uname` gives system-dependent version information. The :mod:`platform` module provides detailed checks for the @@ -1367,8 +1459,8 @@ always available. lazy resolving of symbols when importing a module, if called as ``sys.setdlopenflags(0)``. To share symbols across extension modules, call as ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag values - can be found in the :mod:`os` module (``RTLD_xxx`` constants, e.g. - :data:`os.RTLD_LAZY`). + can be found in the :mod:`os` module (:samp:`RTLD_{xxx}` constants, e.g. + :const:`os.RTLD_LAZY`). .. availability:: Unix. @@ -1397,13 +1489,16 @@ always available. its return value is not used, so it can simply return ``None``. Error in the profile function will cause itself unset. + .. note:: + The same tracing mechanism is used for :func:`!setprofile` as :func:`settrace`. + To trace calls with :func:`!setprofile` inside a tracing function + (e.g. in a debugger breakpoint), see :func:`call_tracing`. + Profile functions should have three arguments: *frame*, *event*, and *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends on the event type. - .. audit-event:: sys.setprofile "" sys.setprofile - The events have the following meaning: ``'call'`` @@ -1425,6 +1520,9 @@ always available. ``'c_exception'`` A C function has raised an exception. *arg* is the C function object. + .. audit-event:: sys.setprofile "" sys.setprofile + + .. function:: setrecursionlimit(limit) Set the maximum depth of the Python interpreter stack to *limit*. This limit @@ -1478,13 +1576,16 @@ always available. function to be used for the new scope, or ``None`` if the scope shouldn't be traced. - The local trace function should return a reference to itself (or to another - function for further tracing in that scope), or ``None`` to turn off tracing - in that scope. + The local trace function should return a reference to itself, or to another + function which would then be used as the local trace function for the scope. If there is any error occurred in the trace function, it will be unset, just like ``settrace(None)`` is called. + .. note:: + Tracing is disabled while calling the trace function (e.g. a function set by + :func:`!settrace`). For recursive tracing see :func:`call_tracing`. + The events have the following meaning: ``'call'`` @@ -1499,7 +1600,8 @@ always available. :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this works. Per-line events may be disabled for a frame by setting - :attr:`f_trace_lines` to :const:`False` on that frame. + :attr:`~frame.f_trace_lines` to :const:`False` on that + :ref:`frame `. ``'return'`` A function (or other code block) is about to return. The local trace @@ -1517,8 +1619,8 @@ always available. opcode details). The local trace function is called; *arg* is ``None``; the return value specifies the new local trace function. Per-opcode events are not emitted by default: they must be explicitly - requested by setting :attr:`f_trace_opcodes` to :const:`True` on the - frame. + requested by setting :attr:`~frame.f_trace_opcodes` to :const:`True` on the + :ref:`frame `. Note that as an exception is propagated down the chain of callers, an ``'exception'`` event is generated at each level. @@ -1547,8 +1649,8 @@ always available. .. versionchanged:: 3.7 - ``'opcode'`` event type added; :attr:`f_trace_lines` and - :attr:`f_trace_opcodes` attributes added to frames + ``'opcode'`` event type added; :attr:`~frame.f_trace_lines` and + :attr:`~frame.f_trace_opcodes` attributes added to frames .. function:: set_asyncgen_hooks(firstiter, finalizer) @@ -1705,7 +1807,7 @@ always available. However, if you are writing a library (and do not control in which context its code will be executed), be aware that the standard streams may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:`~io.BufferedIOBase.buffer` attribute. + do not support the :attr:`!buffer` attribute. .. data:: __stdin__ @@ -1743,7 +1845,7 @@ always available. ``email.mime`` sub-package and the ``email.message`` sub-module are not listed. - See also the :attr:`sys.builtin_module_names` list. + See also the :data:`sys.builtin_module_names` list. .. versionadded:: 3.10 @@ -1753,29 +1855,28 @@ always available. A :term:`named tuple` holding information about the thread implementation. - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +------------------+---------------------------------------------------------+ - | Attribute | Explanation | - +==================+=========================================================+ - | :const:`name` | Name of the thread implementation: | - | | | - | | * ``'nt'``: Windows threads | - | | * ``'pthread'``: POSIX threads | - | | * ``'pthread-stubs'``: stub POSIX threads | - | | (on WebAssembly platforms without threading support) | - | | * ``'solaris'``: Solaris threads | - +------------------+---------------------------------------------------------+ - | :const:`lock` | Name of the lock implementation: | - | | | - | | * ``'semaphore'``: a lock uses a semaphore | - | | * ``'mutex+cond'``: a lock uses a mutex | - | | and a condition variable | - | | * ``None`` if this information is unknown | - +------------------+---------------------------------------------------------+ - | :const:`version` | Name and version of the thread library. It is a string, | - | | or ``None`` if this information is unknown. | - +------------------+---------------------------------------------------------+ + .. attribute:: thread_info.name + + The name of the thread implementation: + + * ``"nt"``: Windows threads + * ``"pthread"``: POSIX threads + * ``"pthread-stubs"``: stub POSIX threads + (on WebAssembly platforms without threading support) + * ``"solaris"``: Solaris threads + + .. attribute:: thread_info.lock + + The name of the lock implementation: + + * ``"semaphore"``: a lock uses a semaphore + * ``"mutex+cond"``: a lock uses a mutex and a condition variable + * ``None`` if this information is unknown + + .. attribute:: thread_info.version + + The name and version of the thread library. + It is a string, or ``None`` if this information is unknown. .. versionadded:: 3.3 @@ -1798,35 +1899,39 @@ always available. The *unraisable* argument has the following attributes: - * *exc_type*: Exception type. - * *exc_value*: Exception value, can be ``None``. - * *exc_traceback*: Exception traceback, can be ``None``. - * *err_msg*: Error message, can be ``None``. - * *object*: Object causing the exception, can be ``None``. + * :attr:`!exc_type`: Exception type. + * :attr:`!exc_value`: Exception value, can be ``None``. + * :attr:`!exc_traceback`: Exception traceback, can be ``None``. + * :attr:`!err_msg`: Error message, can be ``None``. + * :attr:`!object`: Object causing the exception, can be ``None``. - The default hook formats *err_msg* and *object* as: + The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if *err_msg* is ``None``. + if :attr:`!err_msg` is ``None``. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. - Storing *exc_value* using a custom hook can create a reference cycle. It - should be cleared explicitly to break the reference cycle when the - exception is no longer needed. + .. seealso:: + + :func:`excepthook` which handles uncaught exceptions. - Storing *object* using a custom hook can resurrect it if it is set to an - object which is being finalized. Avoid storing *object* after the custom - hook completes to avoid resurrecting objects. + .. warning:: - See also :func:`excepthook` which handles uncaught exceptions. + Storing :attr:`!exc_value` using a custom hook can create a reference cycle. + It should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing :attr:`!object` using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing :attr:`!object` after the custom + hook completes to avoid resurrecting objects. .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook Raise an auditing event ``sys.unraisablehook`` with arguments - ``hook``, ``unraisable`` when an exception that cannot be handled occurs. - The ``unraisable`` object is the same as what will be passed to the hook. - If no hook has been set, ``hook`` may be ``None``. + *hook*, *unraisable* when an exception that cannot be handled occurs. + The *unraisable* object is the same as what will be passed to the hook. + If no hook has been set, *hook* may be ``None``. .. versionadded:: 3.8 @@ -1876,6 +1981,13 @@ always available. .. availability:: Windows. +.. data:: monitoring + :noindex: + + Namespace containing functions and constants for register callbacks + and controlling monitoring events. + See :mod:`sys.monitoring` for details. + .. data:: _xoptions A dictionary of the various implementation-specific flags passed through diff --git a/Doc/library/sys_path_init.rst b/Doc/library/sys_path_init.rst index 72c1387344c2ad..a87a41cf829fa8 100644 --- a/Doc/library/sys_path_init.rst +++ b/Doc/library/sys_path_init.rst @@ -99,8 +99,7 @@ Embedded Python If Python is embedded within another application :c:func:`Py_InitializeFromConfig` and the :c:type:`PyConfig` structure can be used to initialize Python. The path specific -details are described at :ref:`init-path-config`. Alternatively the older :c:func:`Py_SetPath` -can be used to bypass the initialization of the module search path. +details are described at :ref:`init-path-config`. .. seealso:: diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 839c2c015b49ae..2faab212e46eff 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -9,7 +9,7 @@ .. versionadded:: 3.2 -**Source code:** :source:`Lib/sysconfig.py` +**Source code:** :source:`Lib/sysconfig` .. index:: single: configuration information @@ -20,6 +20,7 @@ The :mod:`sysconfig` module provides access to Python's configuration information like the list of installation paths and the configuration variables relevant for the current platform. + Configuration variables ----------------------- @@ -60,6 +61,7 @@ Example of usage:: >>> sysconfig.get_config_vars('AR', 'CXX') ['ar', 'g++'] + .. _installation_paths: Installation paths @@ -68,29 +70,26 @@ Installation paths Python uses an installation scheme that differs depending on the platform and on the installation options. These schemes are stored in :mod:`sysconfig` under unique identifiers based on the value returned by :const:`os.name`. - -Every new component that is installed using :mod:`distutils` or a -Distutils-based system will follow the same scheme to copy its file in the right -places. +The schemes are used by package installers to determine where to copy files to. Python currently supports nine schemes: - *posix_prefix*: scheme for POSIX platforms like Linux or macOS. This is the default scheme used when Python or a component is installed. -- *posix_home*: scheme for POSIX platforms used when a *home* option is used - upon installation. This scheme is used when a component is installed through - Distutils with a specific home prefix. -- *posix_user*: scheme for POSIX platforms used when a component is installed - through Distutils and the *user* option is used. This scheme defines paths - located under the user home directory. +- *posix_home*: scheme for POSIX platforms, when the *home* option is used. + This scheme defines paths located under a specific home prefix. +- *posix_user*: scheme for POSIX platforms, when the *user* option is used. + This scheme defines paths located under the user's home directory + (:const:`site.USER_BASE`). - *posix_venv*: scheme for :mod:`Python virtual environments ` on POSIX - platforms; by default it is the same as *posix_prefix* . -- *nt*: scheme for NT platforms like Windows. -- *nt_user*: scheme for NT platforms, when the *user* option is used. -- *nt_venv*: scheme for :mod:`Python virtual environments ` on NT - platforms; by default it is the same as *nt* . -- *venv*: a scheme with values from ether *posix_venv* or *nt_venv* depending - on the platform Python runs on + platforms; by default it is the same as *posix_prefix*. +- *nt*: scheme for Windows. + This is the default scheme used when Python or a component is installed. +- *nt_user*: scheme for Windows, when the *user* option is used. +- *nt_venv*: scheme for :mod:`Python virtual environments ` on Windows; + by default it is the same as *nt*. +- *venv*: a scheme with values from either *posix_venv* or *nt_venv* depending + on the platform Python runs on. - *osx_framework_user*: scheme for macOS, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique @@ -101,7 +100,7 @@ identifier. Python currently uses eight paths: - *platstdlib*: directory containing the standard Python library files that are platform-specific. - *platlib*: directory for site-specific, platform-specific files. -- *purelib*: directory for site-specific, non-platform-specific files. +- *purelib*: directory for site-specific, non-platform-specific files ('pure' Python). - *include*: directory for non-platform-specific header files for the Python C-API. - *platinclude*: directory for platform-specific header files for @@ -109,7 +108,157 @@ identifier. Python currently uses eight paths: - *scripts*: directory for script files. - *data*: directory for data files. -:mod:`sysconfig` provides some functions to determine these paths. + +.. _sysconfig-user-scheme: + +User scheme +--------------- + +This scheme is designed to be the most convenient solution for users that don't +have write permission to the global site-packages directory or don't want to +install into it. + +Files will be installed into subdirectories of :const:`site.USER_BASE` (written +as :file:`{userbase}` hereafter). This scheme installs pure Python modules and +extension modules in the same location (also known as :const:`site.USER_SITE`). + +``posix_user`` +^^^^^^^^^^^^^^ + +============== =========================================================== +Path Installation directory +============== =========================================================== +*stdlib* :file:`{userbase}/lib/python{X.Y}` +*platstdlib* :file:`{userbase}/lib/python{X.Y}` +*platlib* :file:`{userbase}/lib/python{X.Y}/site-packages` +*purelib* :file:`{userbase}/lib/python{X.Y}/site-packages` +*include* :file:`{userbase}/include/python{X.Y}` +*scripts* :file:`{userbase}/bin` +*data* :file:`{userbase}` +============== =========================================================== + +``nt_user`` +^^^^^^^^^^^ + +============== =========================================================== +Path Installation directory +============== =========================================================== +*stdlib* :file:`{userbase}\\Python{XY}` +*platstdlib* :file:`{userbase}\\Python{XY}` +*platlib* :file:`{userbase}\\Python{XY}\\site-packages` +*purelib* :file:`{userbase}\\Python{XY}\\site-packages` +*include* :file:`{userbase}\\Python{XY}\\Include` +*scripts* :file:`{userbase}\\Python{XY}\\Scripts` +*data* :file:`{userbase}` +============== =========================================================== + +``osx_framework_user`` +^^^^^^^^^^^^^^^^^^^^^^ + +============== =========================================================== +Path Installation directory +============== =========================================================== +*stdlib* :file:`{userbase}/lib/python` +*platstdlib* :file:`{userbase}/lib/python` +*platlib* :file:`{userbase}/lib/python/site-packages` +*purelib* :file:`{userbase}/lib/python/site-packages` +*include* :file:`{userbase}/include/python{X.Y}` +*scripts* :file:`{userbase}/bin` +*data* :file:`{userbase}` +============== =========================================================== + + +.. _sysconfig-home-scheme: + +Home scheme +----------- + +The idea behind the "home scheme" is that you build and maintain a personal +stash of Python modules. This scheme's name is derived from the idea of a +"home" directory on Unix, since it's not unusual for a Unix user to make their +home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. +This scheme can be used by anyone, regardless of the operating system they +are installing for. + +``posix_home`` +^^^^^^^^^^^^^^ + +============== =========================================================== +Path Installation directory +============== =========================================================== +*stdlib* :file:`{home}/lib/python` +*platstdlib* :file:`{home}/lib/python` +*platlib* :file:`{home}/lib/python` +*purelib* :file:`{home}/lib/python` +*include* :file:`{home}/include/python` +*platinclude* :file:`{home}/include/python` +*scripts* :file:`{home}/bin` +*data* :file:`{home}` +============== =========================================================== + + +.. _sysconfig-prefix-scheme: + +Prefix scheme +------------- + +The "prefix scheme" is useful when you wish to use one Python installation to +perform the build/install (i.e., to run the setup script), but install modules +into the third-party module directory of a different Python installation (or +something that looks like a different Python installation). If this sounds a +trifle unusual, it is---that's why the user and home schemes come before. However, +there are at least two known cases where the prefix scheme will be useful. + +First, consider that many Linux distributions put Python in :file:`/usr`, rather +than the more traditional :file:`/usr/local`. This is entirely appropriate, +since in those cases Python is part of "the system" rather than a local add-on. +However, if you are installing Python modules from source, you probably want +them to go in :file:`/usr/local/lib/python2.{X}` rather than +:file:`/usr/lib/python2.{X}`. + +Another possibility is a network filesystem where the name used to write to a +remote directory is different from the name used to read it: for example, the +Python interpreter accessed as :file:`/usr/local/bin/python` might search for +modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to +be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. + +``posix_prefix`` +^^^^^^^^^^^^^^^^ + +============== ========================================================== +Path Installation directory +============== ========================================================== +*stdlib* :file:`{prefix}/lib/python{X.Y}` +*platstdlib* :file:`{prefix}/lib/python{X.Y}` +*platlib* :file:`{prefix}/lib/python{X.Y}/site-packages` +*purelib* :file:`{prefix}/lib/python{X.Y}/site-packages` +*include* :file:`{prefix}/include/python{X.Y}` +*platinclude* :file:`{prefix}/include/python{X.Y}` +*scripts* :file:`{prefix}/bin` +*data* :file:`{prefix}` +============== ========================================================== + +``nt`` +^^^^^^ + +============== ========================================================== +Path Installation directory +============== ========================================================== +*stdlib* :file:`{prefix}\\Lib` +*platstdlib* :file:`{prefix}\\Lib` +*platlib* :file:`{prefix}\\Lib\\site-packages` +*purelib* :file:`{prefix}\\Lib\\site-packages` +*include* :file:`{prefix}\\Include` +*platinclude* :file:`{prefix}\\Include` +*scripts* :file:`{prefix}\\Scripts` +*data* :file:`{prefix}` +============== ========================================================== + + +Installation path functions +--------------------------- + +:mod:`sysconfig` provides some functions to determine these installation paths. .. function:: get_scheme_names() @@ -187,7 +336,7 @@ identifier. Python currently uses eight paths: platform is used. If *vars* is provided, it must be a dictionary of variables that will update - the dictionary return by :func:`get_config_vars`. + the dictionary returned by :func:`get_config_vars`. If *expand* is set to ``False``, the path will not be expanded using the variables. @@ -278,6 +427,7 @@ Other functions Return the path of :file:`Makefile`. +.. _sysconfig-cli: Using :mod:`sysconfig` as a script ---------------------------------- diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index f29ef03267b1ba..b5ab446e0096ed 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -11,12 +11,12 @@ This module provides an interface to the Unix ``syslog`` library routines. Refer to the Unix manual pages for a detailed description of the ``syslog`` facility. +.. availability:: Unix, not Emscripten, not WASI. + This module wraps the system ``syslog`` family of routines. A pure Python library that can speak to a syslog server is available in the :mod:`logging.handlers` module as :class:`SysLogHandler`. -.. include:: ../includes/wasm-notavail.rst - The module defines the following functions: diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 891af1bcf7edff..3e5723a66780ca 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -318,7 +318,7 @@ be finalized; only the internally used file object will be closed. See the .. versionadded:: 3.2 Added support for the context management protocol. -.. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=1) +.. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=1, stream=False) All following arguments are optional and can be accessed as instance attributes as well. @@ -369,6 +369,9 @@ be finalized; only the internally used file object will be closed. See the The *pax_headers* argument is an optional dictionary of strings which will be added as a pax global header if *format* is :const:`PAX_FORMAT`. + If *stream* is set to :const:`True` then while reading the archive info about files + in the archive are not cached, saving memory. + .. versionchanged:: 3.2 Use ``'surrogateescape'`` as the default for the *errors* argument. @@ -378,6 +381,8 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.6 The *name* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Add the *stream* parameter. .. classmethod:: TarFile.open(...) @@ -735,6 +740,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid :type: int @@ -903,7 +913,7 @@ can be: path to where the archive is extracted (i.e. the same path is used for all members):: - filter(/, member: TarInfo, path: str) -> TarInfo | None + filter(member: TarInfo, path: str, /) -> TarInfo | None The callable is called just before each member is extracted, so it can take the current state of the disk into account. @@ -923,17 +933,17 @@ Default named filters The pre-defined, named filters are available as functions, so they can be reused in custom filters: -.. function:: fully_trusted_filter(/, member, path) +.. function:: fully_trusted_filter(member, path) Return *member* unchanged. This implements the ``'fully_trusted'`` filter. -.. function:: tar_filter(/, member, path) +.. function:: tar_filter(member, path) Implements the ``'tar'`` filter. - - Strip leading slashes (``/`` and :attr:`os.sep`) from filenames. + - Strip leading slashes (``/`` and :data:`os.sep`) from filenames. - :ref:`Refuse ` to extract files with absolute paths (in case the name is absolute even after stripping slashes, e.g. ``C:/foo`` on Windows). @@ -942,11 +952,11 @@ reused in custom filters: path (after following symlinks) would end up outside the destination. This raises :class:`~tarfile.OutsideDestinationError`. - Clear high mode bits (setuid, setgid, sticky) and group/other write bits - (:attr:`~stat.S_IWGRP`|:attr:`~stat.S_IWOTH`). + (:const:`~stat.S_IWGRP`|:const:`~stat.S_IWOTH`). Return the modified ``TarInfo`` member. -.. function:: data_filter(/, member, path) +.. function:: data_filter(member, path) Implements the ``'data'`` filter. In addition to what ``tar_filter`` does: @@ -967,10 +977,10 @@ reused in custom filters: - For regular files, including hard links: - Set the owner read and write permissions - (:attr:`~stat.S_IRUSR`|:attr:`~stat.S_IWUSR`). + (:const:`~stat.S_IRUSR`|:const:`~stat.S_IWUSR`). - Remove the group & other executable permission - (:attr:`~stat.S_IXGRP`|:attr:`~stat.S_IXOTH`) - if the owner doesn’t have it (:attr:`~stat.S_IXUSR`). + (:const:`~stat.S_IXGRP`|:const:`~stat.S_IXOTH`) + if the owner doesn’t have it (:const:`~stat.S_IXUSR`). - For other files (directories), set ``mode`` to ``None``, so that extraction methods skip applying permission bits. @@ -1146,31 +1156,31 @@ For a list of the files in a tar archive, use the :option:`-l` option: Command-line options ~~~~~~~~~~~~~~~~~~~~ -.. cmdoption:: -l - --list +.. option:: -l + --list List files in a tarfile. -.. cmdoption:: -c ... - --create ... +.. option:: -c ... + --create ... Create tarfile from source files. -.. cmdoption:: -e [] - --extract [] +.. option:: -e [] + --extract [] Extract tarfile into the current directory if *output_dir* is not specified. -.. cmdoption:: -t - --test +.. option:: -t + --test Test whether the tarfile is valid or not. -.. cmdoption:: -v, --verbose +.. option:: -v, --verbose Verbose output. -.. cmdoption:: --filter +.. option:: --filter Specifies the *filter* for ``--extract``. See :ref:`tarfile-extraction-filter` for details. diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst deleted file mode 100644 index 5a993dc42a5ab2..00000000000000 --- a/Doc/library/telnetlib.rst +++ /dev/null @@ -1,262 +0,0 @@ -:mod:`telnetlib` --- Telnet client -================================== - -.. module:: telnetlib - :synopsis: Telnet client class. - :deprecated: - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/telnetlib.py` - -.. index:: single: protocol; Telnet - -.. deprecated-removed:: 3.11 3.13 - The :mod:`telnetlib` module is deprecated - (see :pep:`PEP 594 <594#telnetlib>` for details and alternatives). - --------------- - -The :mod:`telnetlib` module provides a :class:`Telnet` class that implements the -Telnet protocol. See :rfc:`854` for details about the protocol. In addition, it -provides symbolic constants for the protocol characters (see below), and for the -telnet options. The symbolic names of the telnet options follow the definitions -in ``arpa/telnet.h``, with the leading ``TELOPT_`` removed. For symbolic names -of options which are traditionally not included in ``arpa/telnet.h``, see the -module source itself. - -The symbolic constants for the telnet commands are: IAC, DONT, DO, WONT, WILL, -SE (Subnegotiation End), NOP (No Operation), DM (Data Mark), BRK (Break), IP -(Interrupt process), AO (Abort output), AYT (Are You There), EC (Erase -Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). - -.. include:: ../includes/wasm-notavail.rst - -.. class:: Telnet(host=None, port=0[, timeout]) - - :class:`Telnet` represents a connection to a Telnet server. The instance is - initially not connected by default; the :meth:`~Telnet.open` method must be used to - establish a connection. Alternatively, the host name and optional port - number can be passed to the constructor too, in which case the connection to - the server will be established before the constructor returns. The optional - *timeout* parameter specifies a timeout in seconds for blocking operations - like the connection attempt (if not specified, the global default timeout - setting will be used). - - Do not reopen an already connected instance. - - This class has many :meth:`read_\*` methods. Note that some of them raise - :exc:`EOFError` when the end of the connection is read, because they can return - an empty string for other reasons. See the individual descriptions below. - - A :class:`Telnet` object is a context manager and can be used in a - :keyword:`with` statement. When the :keyword:`!with` block ends, the - :meth:`close` method is called:: - - >>> from telnetlib import Telnet - >>> with Telnet('localhost', 23) as tn: - ... tn.interact() - ... - - .. versionchanged:: 3.6 Context manager support added - - -.. seealso:: - - :rfc:`854` - Telnet Protocol Specification - Definition of the Telnet protocol. - - -.. _telnet-objects: - -Telnet Objects --------------- - -:class:`Telnet` instances have the following methods: - - -.. method:: Telnet.read_until(expected, timeout=None) - - Read until a given byte string, *expected*, is encountered or until *timeout* - seconds have passed. - - When no match is found, return whatever is available instead, possibly empty - bytes. Raise :exc:`EOFError` if the connection is closed and no cooked data - is available. - - -.. method:: Telnet.read_all() - - Read all data until EOF as bytes; block until connection closed. - - -.. method:: Telnet.read_some() - - Read at least one byte of cooked data unless EOF is hit. Return ``b''`` if - EOF is hit. Block if no data is immediately available. - - -.. method:: Telnet.read_very_eager() - - Read everything that can be without blocking in I/O (eager). - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_eager() - - Read readily available data. - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_lazy() - - Process and return data already in the queues (lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. Do not block unless in the - midst of an IAC sequence. - - -.. method:: Telnet.read_very_lazy() - - Return any data available in the cooked queue (very lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. This method never blocks. - - -.. method:: Telnet.read_sb_data() - - Return the data collected between a SB/SE pair (suboption begin/end). The - callback should access these data when it was invoked with a ``SE`` command. - This method never blocks. - - -.. method:: Telnet.open(host, port=0[, timeout]) - - Connect to a host. The optional second argument is the port number, which - defaults to the standard Telnet port (23). The optional *timeout* parameter - specifies a timeout in seconds for blocking operations like the connection - attempt (if not specified, the global default timeout setting will be used). - - Do not try to reopen an already connected instance. - - .. audit-event:: telnetlib.Telnet.open self,host,port telnetlib.Telnet.open - - -.. method:: Telnet.msg(msg, *args) - - Print a debug message when the debug level is ``>`` 0. If extra arguments are - present, they are substituted in the message using the standard string - formatting operator. - - -.. method:: Telnet.set_debuglevel(debuglevel) - - Set the debug level. The higher the value of *debuglevel*, the more debug - output you get (on ``sys.stdout``). - - -.. method:: Telnet.close() - - Close the connection. - - -.. method:: Telnet.get_socket() - - Return the socket object used internally. - - -.. method:: Telnet.fileno() - - Return the file descriptor of the socket object used internally. - - -.. method:: Telnet.write(buffer) - - Write a byte string to the socket, doubling any IAC characters. This can - block if the connection is blocked. May raise :exc:`OSError` if the - connection is closed. - - .. audit-event:: telnetlib.Telnet.write self,buffer telnetlib.Telnet.write - - .. versionchanged:: 3.3 - This method used to raise :exc:`socket.error`, which is now an alias - of :exc:`OSError`. - - -.. method:: Telnet.interact() - - Interaction function, emulates a very dumb Telnet client. - - -.. method:: Telnet.mt_interact() - - Multithreaded version of :meth:`interact`. - - -.. method:: Telnet.expect(list, timeout=None) - - Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either compiled - (:ref:`regex objects `) or uncompiled (byte strings). The - optional second argument is a timeout, in seconds; the default is to block - indefinitely. - - Return a tuple of three items: the index in the list of the first regular - expression that matches; the match object returned; and the bytes read up - till and including the match. - - If end of file is found and no bytes were read, raise :exc:`EOFError`. - Otherwise, when nothing matches, return ``(-1, None, data)`` where *data* is - the bytes received so far (may be empty bytes if a timeout happened). - - If a regular expression ends with a greedy match (such as ``.*``) or if more - than one expression can match the same input, the results are - non-deterministic, and may depend on the I/O timing. - - -.. method:: Telnet.set_option_negotiation_callback(callback) - - Each time a telnet option is read on the input flow, this *callback* (if set) is - called with the following parameters: callback(telnet socket, command - (DO/DONT/WILL/WONT), option). No other action is done afterwards by telnetlib. - - -.. _telnet-example: - -Telnet Example --------------- - -.. sectionauthor:: Peter Funk - - -A simple example illustrating typical use:: - - import getpass - import telnetlib - - HOST = "localhost" - user = input("Enter your remote account: ") - password = getpass.getpass() - - tn = telnetlib.Telnet(HOST) - - tn.read_until(b"login: ") - tn.write(user.encode('ascii') + b"\n") - if password: - tn.read_until(b"Password: ") - tn.write(password.encode('ascii') + b"\n") - - tn.write(b"ls\n") - tn.write(b"exit\n") - - print(tn.read_all().decode('ascii')) - diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index fd4c294613fd31..b68a78e8267bcc 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -59,7 +59,7 @@ The module defines the following user-callable items: platforms, it is a file-like object whose :attr:`!file` attribute is the underlying true file object. - The :py:data:`os.O_TMPFILE` flag is used if it is available and works + The :py:const:`os.O_TMPFILE` flag is used if it is available and works (Linux-specific, requires Linux kernel 3.11 or later). On platforms that are neither Posix nor Cygwin, TemporaryFile is an alias @@ -69,7 +69,7 @@ The module defines the following user-callable items: .. versionchanged:: 3.5 - The :py:data:`os.O_TMPFILE` flag is now used if available. + The :py:const:`os.O_TMPFILE` flag is now used if available. .. versionchanged:: 3.8 Added *errors* parameter. @@ -115,14 +115,14 @@ The module defines the following user-callable items: * On Windows, make sure that at least one of the following conditions are fulfilled: - * *delete* is false - * additional open shares delete access (e.g. by calling :func:`os.open` - with the flag ``O_TEMPORARY``) - * *delete* is true but *delete_on_close* is false. Note, that in this - case the additional opens that do not share delete access (e.g. - created via builtin :func:`open`) must be closed before exiting the - context manager, else the :func:`os.unlink` call on context manager - exit will fail with a :exc:`PermissionError`. + * *delete* is false + * additional open shares delete access (e.g. by calling :func:`os.open` + with the flag ``O_TEMPORARY``) + * *delete* is true but *delete_on_close* is false. Note, that in this + case the additional opens that do not share delete access (e.g. + created via builtin :func:`open`) must be closed before exiting the + context manager, else the :func:`os.unlink` call on context manager + exit will fail with a :exc:`PermissionError`. On Windows, if *delete_on_close* is false, and the file is created in a directory for which the user lacks delete access, then the :func:`os.unlink` @@ -147,7 +147,7 @@ The module defines the following user-callable items: This class operates exactly as :func:`TemporaryFile` does, except that data is spooled in memory until the file size exceeds *max_size*, or - until the file's :func:`fileno` method is called, at which point the + until the file's :func:`~io.IOBase.fileno` method is called, at which point the contents are written to disk and operation proceeds as with :func:`TemporaryFile`. @@ -404,13 +404,13 @@ Here are some examples of typical usage of the :mod:`tempfile` module:: # create a temporary file using a context manager # close the file, use the name to open the file again - >>> with tempfile.TemporaryFile(delete_on_close=False) as fp: - ... fp.write(b'Hello world!') - ... fp.close() - # the file is closed, but not removed - # open the file again by using its name - ... with open(fp.name) as f - ... f.read() + >>> with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + ... fp.write(b'Hello world!') + ... fp.close() + ... # the file is closed, but not removed + ... # open the file again by using its name + ... with open(fp.name, mode='rb') as f: + ... f.read() b'Hello world!' >>> # file is now removed diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index fb1ff567d49e5c..57705ddc4e6470 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -16,6 +16,8 @@ complete description of these calls, see :manpage:`termios(3)` Unix manual page. It is only available for those Unix versions that support POSIX *termios* style tty I/O control configured during installation. +.. availability:: Unix. + All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by ``sys.stdin.fileno()``, or a :term:`file object`, such as ``sys.stdin`` itself. @@ -43,10 +45,20 @@ The module defines the following functions: Set the tty attributes for file descriptor *fd* from the *attributes*, which is a list like the one returned by :func:`tcgetattr`. The *when* argument - determines when the attributes are changed: :const:`TCSANOW` to change - immediately, :const:`TCSADRAIN` to change after transmitting all queued output, - or :const:`TCSAFLUSH` to change after transmitting all queued output and - discarding all queued input. + determines when the attributes are changed: + + .. data:: TCSANOW + + Change attributes immediately. + + .. data:: TCSADRAIN + + Change attributes after transmitting all queued output. + + .. data:: TCSAFLUSH + + Change attributes after transmitting all queued output and + discarding all queued input. .. function:: tcsendbreak(fd, duration) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 20f633b8f569be..6cbdc39b3c024d 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -159,6 +159,9 @@ guidelines to be followed: Running tests using the command-line interface ---------------------------------------------- +.. module:: test.regrtest + :synopsis: Drives the regression test suite. + The :mod:`test` package can be run as a script to drive Python's regression test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under the hood, it uses :mod:`test.regrtest`; the call :program:`python -m @@ -472,7 +475,7 @@ The :mod:`test.support` module defines the following functions: .. function:: with_pymalloc() - Return :data:`_testcapi.WITH_PYMALLOC`. + Return :const:`_testcapi.WITH_PYMALLOC`. .. function:: requires(resource, msg=None) @@ -498,44 +501,6 @@ The :mod:`test.support` module defines the following functions: rather than looking directly in the path directories. -.. function:: match_test(test) - - Determine whether *test* matches the patterns set in :func:`set_match_tests`. - - -.. function:: set_match_tests(accept_patterns=None, ignore_patterns=None) - - Define match patterns on test filenames and test method names for filtering tests. - - -.. function:: run_unittest(*classes) - - Execute :class:`unittest.TestCase` subclasses passed to the function. The - function scans the classes for methods starting with the prefix ``test_`` - and executes the tests individually. - - It is also legal to pass strings as parameters; these should be keys in - ``sys.modules``. Each associated module will be scanned by - ``unittest.TestLoader.loadTestsFromModule()``. This is usually seen in the - following :func:`test_main` function:: - - def test_main(): - support.run_unittest(__name__) - - This will run all tests defined in the named module. - - -.. function:: run_doctest(module, verbosity=None, optionflags=0) - - Run :func:`doctest.testmod` on the given *module*. Return - ``(failure_count, test_count)``. - - If *verbosity* is ``None``, :func:`doctest.testmod` is run with verbosity - set to :data:`verbose`. Otherwise, it is run with verbosity set to - ``None``. *optionflags* is passed as ``optionflags`` to - :func:`doctest.testmod`. - - .. function:: get_pagesize() Get size of a page in bytes. @@ -803,7 +768,7 @@ The :mod:`test.support` module defines the following functions: .. decorator:: requires_limited_api - Decorator for only running the test if :ref:`Limited C API ` + Decorator for only running the test if :ref:`Limited C API ` is available. @@ -1040,10 +1005,10 @@ The :mod:`test.support` module defines the following classes: `SetErrorMode `_. On UNIX, :func:`resource.setrlimit` is used to set - :attr:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file + :const:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file creation. - On both platforms, the old value is restored by :meth:`__exit__`. + On both platforms, the old value is restored by :meth:`~object.__exit__`. .. class:: SaveSignals() diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index 1a9d5f98f78a7e..7445410f91808c 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -60,7 +60,7 @@ functions should be good enough; otherwise, you should use an instance of First the whitespace in *text* is collapsed (all whitespace is replaced by single spaces). If the result fits in the *width*, it is returned. Otherwise, enough words are dropped from the end so that the remaining words - plus the :attr:`placeholder` fit within :attr:`width`:: + plus the *placeholder* fit within *width*:: >>> textwrap.shorten("Hello world!", width=12) 'Hello world!' @@ -173,7 +173,7 @@ hyphenated words; only then will long words be broken if necessary, unless .. attribute:: expand_tabs (default: ``True``) If true, then all tab characters in *text* will be - expanded to spaces using the :meth:`expandtabs` method of *text*. + expanded to spaces using the :meth:`~str.expandtabs` method of *text*. .. attribute:: tabsize @@ -238,7 +238,7 @@ hyphenated words; only then will long words be broken if necessary, unless However, the sentence detection algorithm is imperfect: it assumes that a sentence ending consists of a lowercase letter followed by one of ``'.'``, ``'!'``, or ``'?'``, possibly followed by one of ``'"'`` or ``"'"``, - followed by a space. One problem with this is algorithm is that it is + followed by a space. One problem with this algorithm is that it is unable to detect the difference between "Dr." in :: [...] Dr. Frankenstein's monster [...] diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 83ed48052704fb..b85b7f008d1594 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -127,10 +127,13 @@ This module defines the following functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. .. versionadded:: 3.8 + .. versionchanged:: 3.13 + Added support for GNU/kFreeBSD. + .. function:: enumerate() @@ -272,7 +275,7 @@ The instance's values will be different for separate threads. A class that represents thread-local data. For more details and extensive examples, see the documentation string of the - :mod:`_threading_local` module: :source:`Lib/_threading_local.py`. + :mod:`!_threading_local` module: :source:`Lib/_threading_local.py`. .. _thread-objects: @@ -285,7 +288,7 @@ thread of control. There are two ways to specify the activity: by passing a callable object to the constructor, or by overriding the :meth:`~Thread.run` method in a subclass. No other methods (except for the constructor) should be overridden in a subclass. In other words, *only* override the -:meth:`~Thread.__init__` and :meth:`~Thread.run` methods of this class. +``__init__()`` and :meth:`~Thread.run` methods of this class. Once a thread object is created, its activity must be started by calling the thread's :meth:`~Thread.start` method. This invokes the :meth:`~Thread.run` @@ -337,7 +340,7 @@ since it is impossible to detect the termination of alien threads. are: *group* should be ``None``; reserved for future extension when a - :class:`ThreadGroup` class is implemented. + :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. Defaults to ``None``, meaning nothing is called. @@ -1009,7 +1012,7 @@ This class represents an action that should be run only after a certain amount of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` and as such also functions as an example of creating custom threads. -Timers are started, as with threads, by calling their :meth:`~Timer.start` +Timers are started, as with threads, by calling their :meth:`Timer.start ` method. The timer can be stopped (before its action has begun) by calling the :meth:`~Timer.cancel` method. The interval the timer will wait before executing its action may not be exactly the same as the interval specified by @@ -1147,10 +1150,10 @@ As an example, here is a simple way to synchronize a client and server thread:: Using locks, conditions, and semaphores in the :keyword:`!with` statement ------------------------------------------------------------------------- -All of the objects provided by this module that have :meth:`acquire` and -:meth:`release` methods can be used as context managers for a :keyword:`with` -statement. The :meth:`acquire` method will be called when the block is -entered, and :meth:`release` will be called when the block is exited. Hence, +All of the objects provided by this module that have ``acquire`` and +``release`` methods can be used as context managers for a :keyword:`with` +statement. The ``acquire`` method will be called when the block is +entered, and ``release`` will be called when the block is exited. Hence, the following snippet:: with some_lock: diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 9f23a6fc7d5341..577600881676b3 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -71,8 +71,8 @@ An explanation of some terminology and conventions is in order. * On the other hand, the precision of :func:`.time` and :func:`sleep` is better than their Unix equivalents: times are expressed as floating point numbers, :func:`.time` returns the most accurate time available (using Unix - :c:func:`gettimeofday` where available), and :func:`sleep` will accept a time - with a nonzero fraction (Unix :c:func:`select` is used to implement this, where + :c:func:`!gettimeofday` where available), and :func:`sleep` will accept a time + with a nonzero fraction (Unix :c:func:`!select` is used to implement this, where available). * The time value as returned by :func:`gmtime`, :func:`localtime`, and @@ -84,12 +84,14 @@ An explanation of some terminology and conventions is in order. See :class:`struct_time` for a description of these objects. .. versionchanged:: 3.3 - The :class:`struct_time` type was extended to provide the :attr:`tm_gmtoff` - and :attr:`tm_zone` attributes when platform supports corresponding + The :class:`struct_time` type was extended to provide + the :attr:`~struct_time.tm_gmtoff` and :attr:`~struct_time.tm_zone` + attributes when platform supports corresponding ``struct tm`` members. .. versionchanged:: 3.6 - The :class:`struct_time` attributes :attr:`tm_gmtoff` and :attr:`tm_zone` + The :class:`struct_time` attributes + :attr:`~struct_time.tm_gmtoff` and :attr:`~struct_time.tm_zone` are now available on all platforms. * Use the following functions to convert between time representations: @@ -379,6 +381,8 @@ Functions * Or use ``nanosleep()`` if available (resolution: 1 nanosecond); * Or use ``select()`` (resolution: 1 microsecond). + .. audit-event:: time.sleep secs + .. versionchanged:: 3.11 On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now used if available. On Windows, a waitable timer is now used. @@ -389,6 +393,9 @@ Functions :pep:`475` for the rationale). + .. versionchanged:: 3.13 + Raises an auditing event. + .. index:: single: % (percent); datetime format @@ -496,6 +503,8 @@ Functions When used with the :func:`strptime` function, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. + .. _leap-second: + (2) The range really is ``0`` to ``61``; value ``60`` is valid in timestamps representing `leap seconds`_ and value ``61`` is supported @@ -566,32 +575,55 @@ Functions tuple` interface: values can be accessed by index and by attribute name. The following values are present: - +-------+-------------------+---------------------------------+ - | Index | Attribute | Values | - +=======+===================+=================================+ - | 0 | :attr:`tm_year` | (for example, 1993) | - +-------+-------------------+---------------------------------+ - | 1 | :attr:`tm_mon` | range [1, 12] | - +-------+-------------------+---------------------------------+ - | 2 | :attr:`tm_mday` | range [1, 31] | - +-------+-------------------+---------------------------------+ - | 3 | :attr:`tm_hour` | range [0, 23] | - +-------+-------------------+---------------------------------+ - | 4 | :attr:`tm_min` | range [0, 59] | - +-------+-------------------+---------------------------------+ - | 5 | :attr:`tm_sec` | range [0, 61]; see **(2)** in | - | | | :func:`strftime` description | - +-------+-------------------+---------------------------------+ - | 6 | :attr:`tm_wday` | range [0, 6], Monday is 0 | - +-------+-------------------+---------------------------------+ - | 7 | :attr:`tm_yday` | range [1, 366] | - +-------+-------------------+---------------------------------+ - | 8 | :attr:`tm_isdst` | 0, 1 or -1; see below | - +-------+-------------------+---------------------------------+ - | N/A | :attr:`tm_zone` | abbreviation of timezone name | - +-------+-------------------+---------------------------------+ - | N/A | :attr:`tm_gmtoff` | offset east of UTC in seconds | - +-------+-------------------+---------------------------------+ + .. list-table:: + + * - Index + - Attribute + - Values + + * - 0 + - .. attribute:: tm_year + - (for example, 1993) + + * - 1 + - .. attribute:: tm_mon + - range [1, 12] + + * - 2 + - .. attribute:: tm_day + - range [1, 31] + + * - 3 + - .. attribute:: tm_hour + - range [0, 23] + + * - 4 + - .. attribute:: tm_min + - range [0, 59] + + * - 5 + - .. attribute:: tm_sec + - range [0, 61]; see :ref:`Note (2) ` in :func:`strftime` + + * - 6 + - .. attribute:: tm_wday + - range [0, 6]; Monday is 0 + + * - 7 + - .. attribute:: tm_yday + - range [1, 366] + + * - 8 + - .. attribute:: tm_isdst + - 0, 1 or -1; see below + + * - N/A + - .. attribute:: tm_zone + - abbreviation of timezone name + + * - N/A + - .. attribute:: tm_gmtoff + - offset east of UTC in seconds Note that unlike the C structure, the month value is a range of [1, 12], not [0, 11]. @@ -912,8 +944,8 @@ Timezone Constants For the above Timezone constants (:data:`altzone`, :data:`daylight`, :data:`timezone`, and :data:`tzname`), the value is determined by the timezone rules in effect at module load time or the last time :func:`tzset` is called and may be incorrect - for times in the past. It is recommended to use the :attr:`tm_gmtoff` and - :attr:`tm_zone` results from :func:`localtime` to obtain timezone information. + for times in the past. It is recommended to use the :attr:`~struct_time.tm_gmtoff` and + :attr:`~struct_time.tm_zone` results from :func:`localtime` to obtain timezone information. .. seealso:: diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 32ab565aba0c08..616f8365b80f6c 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -27,11 +27,11 @@ can be used to compare three different expressions: .. code-block:: shell-session - $ python -m timeit '"-".join(str(n) for n in range(100))' + $ python -m timeit "'-'.join(str(n) for n in range(100))" 10000 loops, best of 5: 30.2 usec per loop - $ python -m timeit '"-".join([str(n) for n in range(100)])' + $ python -m timeit "'-'.join([str(n) for n in range(100)])" 10000 loops, best of 5: 27.5 usec per loop - $ python -m timeit '"-".join(map(str, range(100)))' + $ python -m timeit "'-'.join(map(str, range(100)))" 10000 loops, best of 5: 23.2 usec per loop This can be achieved from the :ref:`python-interface` with:: @@ -86,9 +86,11 @@ The module defines three convenience functions and a public class: .. versionchanged:: 3.7 Default value of *repeat* changed from 3 to 5. + .. function:: default_timer() - The default timer, which is always :func:`time.perf_counter`. + The default timer, which is always time.perf_counter(), returns float seconds. + An alternative, time.perf_counter_ns, returns integer nanoseconds. .. versionchanged:: 3.3 :func:`time.perf_counter` is now the default timer. @@ -124,7 +126,7 @@ The module defines three convenience functions and a public class: Time *number* executions of the main statement. This executes the setup statement once, and then returns the time it takes to execute the main - statement a number of times, measured in seconds as a float. + statement a number of times. The default timer returns seconds as a float. The argument is the number of times through the loop, defaulting to one million. The main statement, the setup statement and the timer function to be used are passed to the constructor. @@ -149,7 +151,7 @@ The module defines three convenience functions and a public class: so that the total time >= 0.2 second, returning the eventual (number of loops, time taken for that number of loops). It calls :meth:`.timeit` with increasing numbers from the sequence 1, 2, 5, - 10, 20, 50, ... until the time taken is at least 0.2 second. + 10, 20, 50, ... until the time taken is at least 0.2 seconds. If *callback* is given and is not ``None``, it will be called after each trial with two arguments: ``callback(number, time_taken)``. @@ -212,36 +214,36 @@ Where the following options are understood: .. program:: timeit -.. cmdoption:: -n N, --number=N +.. option:: -n N, --number=N how many times to execute 'statement' -.. cmdoption:: -r N, --repeat=N +.. option:: -r N, --repeat=N how many times to repeat the timer (default 5) -.. cmdoption:: -s S, --setup=S +.. option:: -s S, --setup=S statement to be executed once initially (default ``pass``) -.. cmdoption:: -p, --process +.. option:: -p, --process measure process time, not wallclock time, using :func:`time.process_time` instead of :func:`time.perf_counter`, which is the default .. versionadded:: 3.3 -.. cmdoption:: -u, --unit=U +.. option:: -u, --unit=U specify a time unit for timer output; can select ``nsec``, ``usec``, ``msec``, or ``sec`` .. versionadded:: 3.5 -.. cmdoption:: -v, --verbose +.. option:: -v, --verbose print raw timing results; repeat for more digits precision -.. cmdoption:: -h, --help +.. option:: -h, --help print a short usage message and exit @@ -277,9 +279,9 @@ It is possible to provide a setup statement that is executed only once at the be .. code-block:: shell-session - $ python -m timeit -s 'text = "sample string"; char = "g"' 'char in text' + $ python -m timeit -s "text = 'sample string'; char = 'g'" "char in text" 5000000 loops, best of 5: 0.0877 usec per loop - $ python -m timeit -s 'text = "sample string"; char = "g"' 'text.find(char)' + $ python -m timeit -s "text = 'sample string'; char = 'g'" "text.find(char)" 1000000 loops, best of 5: 0.342 usec per loop In the output, there are three fields. The loop count, which tells you how many @@ -313,14 +315,14 @@ to test for missing and present object attributes: .. code-block:: shell-session - $ python -m timeit 'try:' ' str.__bool__' 'except AttributeError:' ' pass' + $ python -m timeit "try:" " str.__bool__" "except AttributeError:" " pass" 20000 loops, best of 5: 15.7 usec per loop - $ python -m timeit 'if hasattr(str, "__bool__"): pass' + $ python -m timeit "if hasattr(str, '__bool__'): pass" 50000 loops, best of 5: 4.26 usec per loop - $ python -m timeit 'try:' ' int.__bool__' 'except AttributeError:' ' pass' + $ python -m timeit "try:" " int.__bool__" "except AttributeError:" " pass" 200000 loops, best of 5: 1.43 usec per loop - $ python -m timeit 'if hasattr(int, "__bool__"): pass' + $ python -m timeit "if hasattr(int, '__bool__'): pass" 100000 loops, best of 5: 2.23 usec per loop :: diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst index 3dc2130539c2cf..0593f8b73ea545 100644 --- a/Doc/library/tk.rst +++ b/Doc/library/tk.rst @@ -12,8 +12,7 @@ Graphical User Interfaces with Tk Tk/Tcl has long been an integral part of Python. It provides a robust and platform independent windowing toolkit, that is available to Python programmers -using the :mod:`tkinter` package, and its extension, the :mod:`tkinter.tix` and -the :mod:`tkinter.ttk` modules. +using the :mod:`tkinter` package, and its extension, the :mod:`tkinter.ttk` module. The :mod:`tkinter` package is a thin object-oriented layer on top of Tcl/Tk. To use :mod:`tkinter`, you don't need to write Tcl code, but you will need to @@ -39,7 +38,6 @@ alternative `GUI frameworks and tools ` of the default button + for this message window (:data:`OK`, :data:`CANCEL`, and so on). + If this option is not specified, the first button in the dialog will + be made the default. + + *detail* + Specifies an auxiliary message to the main message given by the + *message* option. + The message detail will be presented beneath the main message and, + where supported by the OS, in a less emphasized font than the main + message. + + *icon* + Specifies an :ref:`icon ` to display. + If this option is not specified, then the :data:`INFO` icon will be + displayed. + + *message* + Specifies the message to display in this message box. + The default value is an empty string. + + *parent* + Makes the specified window the logical parent of the message box. + The message box is displayed on top of its parent window. + + *title* + Specifies a string to display as the title of the message box. + This option is ignored on macOS, where platform guidelines forbid + the use of a title on this kind of dialog. + + *type* + Arranges for a :ref:`predefined set of buttons ` + to be displayed. + + .. method:: show(**options) + + Display a message window and wait for the user to select one of the buttons. Then return the symbolic name of the selected button. + Keyword arguments can override options specified in the constructor. + **Information message box** -.. method:: showinfo(title=None, message=None, **options) +.. function:: showinfo(title=None, message=None, **options) + + Creates and displays an information message box with the specified title + and message. **Warning message boxes** -.. method:: showwarning(title=None, message=None, **options) - showerror(title=None, message=None, **options) +.. function:: showwarning(title=None, message=None, **options) + + Creates and displays a warning message box with the specified title + and message. + +.. function:: showerror(title=None, message=None, **options) + + Creates and displays an error message box with the specified title + and message. **Question message boxes** -.. method:: askquestion(title=None, message=None, **options) - askokcancel(title=None, message=None, **options) - askretrycancel(title=None, message=None, **options) - askyesno(title=None, message=None, **options) - askyesnocancel(title=None, message=None, **options) +.. function:: askquestion(title=None, message=None, *, type=YESNO, **options) + + Ask a question. By default shows buttons :data:`YES` and :data:`NO`. + Returns the symbolic name of the selected button. + +.. function:: askokcancel(title=None, message=None, **options) + + Ask if operation should proceed. Shows buttons :data:`OK` and :data:`CANCEL`. + Returns ``True`` if the answer is ok and ``False`` otherwise. + +.. function:: askretrycancel(title=None, message=None, **options) + + Ask if operation should be retried. Shows buttons :data:`RETRY` and :data:`CANCEL`. + Return ``True`` if the answer is yes and ``False`` otherwise. + +.. function:: askyesno(title=None, message=None, **options) + + Ask a question. Shows buttons :data:`YES` and :data:`NO`. + Returns ``True`` if the answer is yes and ``False`` otherwise. + +.. function:: askyesnocancel(title=None, message=None, **options) + + Ask a question. Shows buttons :data:`YES`, :data:`NO` and :data:`CANCEL`. + Return ``True`` if the answer is yes, ``None`` if cancelled, and ``False`` + otherwise. + + +.. _messagebox-buttons: + +Symbolic names of buttons: + +.. data:: ABORT + :value: 'abort' +.. data:: RETRY + :value: 'retry' +.. data:: IGNORE + :value: 'ignore' +.. data:: OK + :value: 'ok' +.. data:: CANCEL + :value: 'cancel' +.. data:: YES + :value: 'yes' +.. data:: NO + :value: 'no' + +.. _messagebox-types: + +Predefined sets of buttons: + +.. data:: ABORTRETRYIGNORE + :value: 'abortretryignore' + + Displays three buttons whose symbolic names are :data:`ABORT`, + :data:`RETRY` and :data:`IGNORE`. + +.. data:: OK + :value: 'ok' + :noindex: + + Displays one button whose symbolic name is :data:`OK`. + +.. data:: OKCANCEL + :value: 'okcancel' + + Displays two buttons whose symbolic names are :data:`OK` and + :data:`CANCEL`. + +.. data:: RETRYCANCEL + :value: 'retrycancel' + + Displays two buttons whose symbolic names are :data:`RETRY` and + :data:`CANCEL`. + +.. data:: YESNO + :value: 'yesno' + + Displays two buttons whose symbolic names are :data:`YES` and + :data:`NO`. + +.. data:: YESNOCANCEL + :value: 'yesnocancel' + + Displays three buttons whose symbolic names are :data:`YES`, + :data:`NO` and :data:`CANCEL`. + +.. _messagebox-icons: + +Icon images: + +.. data:: ERROR + :value: 'error' +.. data:: INFO + :value: 'info' +.. data:: QUESTION + :value: 'question' +.. data:: WARNING + :value: 'warning' diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index c8e4317be75879..e084d8554c7c09 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -58,8 +58,8 @@ details that are unchanged. * `Modern Tkinter for Busy Python Developers `_ By Mark Roseman. (ISBN 978-1999149567) - * `Python and Tkinter Programming `_ - By Alan Moore. (ISBN 978-1788835886) + * `Python GUI programming with Tkinter `_ + By Alan D. Moore. (ISBN 978-1788835886) * `Programming Python `_ By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101) @@ -163,7 +163,7 @@ the modern themed widget set and API:: interpreter and calls :func:`exec` on the contents of :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the profile files is the :envvar:`HOME` environment variable or, if that - isn't defined, then :attr:`os.curdir`. + isn't defined, then :data:`os.curdir`. .. attribute:: tk @@ -232,6 +232,9 @@ The modules that provide Tk support include: Additional modules: +.. module:: _tkinter + :synopsis: A binary module that contains the low-level interface to Tcl/Tk. + :mod:`_tkinter` A binary module that contains the low-level interface to Tcl/Tk. It is automatically imported by the main :mod:`tkinter` module, @@ -252,10 +255,6 @@ Additional modules: (experimental) Drag-and-drop support for :mod:`tkinter`. This will become deprecated when it is replaced with the Tk DND. -:mod:`tkinter.tix` - (deprecated) An older third-party Tcl/Tk package that adds several new - widgets. Better alternatives for most can be found in :mod:`tkinter.ttk`. - :mod:`turtle` Turtle graphics in a Tk window. @@ -352,7 +351,7 @@ Understanding How Tkinter Wraps Tcl/Tk When your application uses Tkinter's classes and methods, internally Tkinter is assembling strings representing Tcl/Tk commands, and executing those -commands in the Tcl interpreter attached to your applicaton's :class:`Tk` +commands in the Tcl interpreter attached to your application's :class:`Tk` instance. Whether it's trying to navigate reference documentation, trying to find @@ -533,24 +532,24 @@ interpreter will fail. A number of special cases exist: - * Tcl/Tk libraries can be built so they are not thread-aware. In this case, - :mod:`tkinter` calls the library from the originating Python thread, even - if this is different than the thread that created the Tcl interpreter. A global - lock ensures only one call occurs at a time. +* Tcl/Tk libraries can be built so they are not thread-aware. In this case, + :mod:`tkinter` calls the library from the originating Python thread, even + if this is different than the thread that created the Tcl interpreter. A global + lock ensures only one call occurs at a time. - * While :mod:`tkinter` allows you to create more than one instance of a :class:`Tk` - object (with its own interpreter), all interpreters that are part of the same - thread share a common event queue, which gets ugly fast. In practice, don't create - more than one instance of :class:`Tk` at a time. Otherwise, it's best to create - them in separate threads and ensure you're running a thread-aware Tcl/Tk build. +* While :mod:`tkinter` allows you to create more than one instance of a :class:`Tk` + object (with its own interpreter), all interpreters that are part of the same + thread share a common event queue, which gets ugly fast. In practice, don't create + more than one instance of :class:`Tk` at a time. Otherwise, it's best to create + them in separate threads and ensure you're running a thread-aware Tcl/Tk build. - * Blocking event handlers are not the only way to prevent the Tcl interpreter from - reentering the event loop. It is even possible to run multiple nested event loops - or abandon the event loop entirely. If you're doing anything tricky when it comes - to events or threads, be aware of these possibilities. +* Blocking event handlers are not the only way to prevent the Tcl interpreter from + reentering the event loop. It is even possible to run multiple nested event loops + or abandon the event loop entirely. If you're doing anything tricky when it comes + to events or threads, be aware of these possibilities. - * There are a few select :mod:`tkinter` functions that presently work only when - called from the thread that created the Tcl interpreter. +* There are a few select :mod:`tkinter` functions that presently work only when + called from the thread that created the Tcl interpreter. Handy Reference diff --git a/Doc/library/tkinter.tix.rst b/Doc/library/tkinter.tix.rst deleted file mode 100644 index c86fcfa6a3f46d..00000000000000 --- a/Doc/library/tkinter.tix.rst +++ /dev/null @@ -1,583 +0,0 @@ -:mod:`tkinter.tix` --- Extension widgets for Tk -=============================================== - -.. module:: tkinter.tix - :synopsis: Tk Extension Widgets for Tkinter - -.. sectionauthor:: Mike Clarkson - -**Source code:** :source:`Lib/tkinter/tix.py` - -.. index:: single: Tix - -.. deprecated:: 3.6 - This Tk extension is unmaintained and should not be used in new code. Use - :mod:`tkinter.ttk` instead. - --------------- - -The :mod:`tkinter.tix` (Tk Interface Extension) module provides an additional -rich set of widgets. Although the standard Tk library has many useful widgets, -they are far from complete. The :mod:`tkinter.tix` library provides most of the -commonly needed widgets that are missing from standard Tk: :class:`HList`, -:class:`ComboBox`, :class:`Control` (a.k.a. SpinBox) and an assortment of -scrollable widgets. -:mod:`tkinter.tix` also includes many more widgets that are generally useful in -a wide range of applications: :class:`NoteBook`, :class:`FileEntry`, -:class:`PanedWindow`, etc; there are more than 40 of them. - -With all these new widgets, you can introduce new interaction techniques into -applications, creating more useful and more intuitive user interfaces. You can -design your application by choosing the most appropriate widgets to match the -special needs of your application and users. - -.. seealso:: - - `Tix Homepage `_ - The home page for :mod:`Tix`. This includes links to additional documentation - and downloads. - - `Tix Man Pages `_ - On-line version of the man pages and reference material. - - `Tix Programming Guide `_ - On-line version of the programmer's reference material. - - `Tix Development Applications `_ - Tix applications for development of Tix and Tkinter programs. Tide applications - work under Tk or Tkinter, and include :program:`TixInspect`, an inspector to - remotely modify and debug Tix/Tk/Tkinter applications. - - -Using Tix ---------- - - -.. class:: Tk(screenName=None, baseName=None, className='Tix') - - Toplevel widget of Tix which represents mostly the main window of an - application. It has an associated Tcl interpreter. - - Classes in the :mod:`tkinter.tix` module subclasses the classes in the - :mod:`tkinter`. The former imports the latter, so to use :mod:`tkinter.tix` - with Tkinter, all you need to do is to import one module. In general, you - can just import :mod:`tkinter.tix`, and replace the toplevel call to - :class:`tkinter.Tk` with :class:`tix.Tk`:: - - from tkinter import tix - from tkinter.constants import * - root = tix.Tk() - -To use :mod:`tkinter.tix`, you must have the Tix widgets installed, usually -alongside your installation of the Tk widgets. To test your installation, try -the following:: - - from tkinter import tix - root = tix.Tk() - root.tk.eval('package require Tix') - - -Tix Widgets ------------ - -`Tix `_ -introduces over 40 widget classes to the :mod:`tkinter` repertoire. - - -Basic Widgets -^^^^^^^^^^^^^ - - -.. class:: Balloon() - - A `Balloon - `_ that - pops up over a widget to provide help. When the user moves the cursor inside a - widget to which a Balloon widget has been bound, a small pop-up window with a - descriptive message will be shown on the screen. - -.. Python Demo of: -.. \ulink{Balloon}{https://tix.sourceforge.net/dist/current/demos/samples/Balloon.tcl} - - -.. class:: ButtonBox() - - The `ButtonBox - `_ - widget creates a box of buttons, such as is commonly used for ``Ok Cancel``. - -.. Python Demo of: -.. \ulink{ButtonBox}{https://tix.sourceforge.net/dist/current/demos/samples/BtnBox.tcl} - - -.. class:: ComboBox() - - The `ComboBox - `_ - widget is similar to the combo box control in MS Windows. The user can select a - choice by either typing in the entry subwidget or selecting from the listbox - subwidget. - -.. Python Demo of: -.. \ulink{ComboBox}{https://tix.sourceforge.net/dist/current/demos/samples/ComboBox.tcl} - - -.. class:: Control() - - The `Control - `_ - widget is also known as the :class:`SpinBox` widget. The user can adjust the - value by pressing the two arrow buttons or by entering the value directly into - the entry. The new value will be checked against the user-defined upper and - lower limits. - -.. Python Demo of: -.. \ulink{Control}{https://tix.sourceforge.net/dist/current/demos/samples/Control.tcl} - - -.. class:: LabelEntry() - - The `LabelEntry - `_ - widget packages an entry widget and a label into one mega widget. It can - be used to simplify the creation of "entry-form" type of interface. - -.. Python Demo of: -.. \ulink{LabelEntry}{https://tix.sourceforge.net/dist/current/demos/samples/LabEntry.tcl} - - -.. class:: LabelFrame() - - The `LabelFrame - `_ - widget packages a frame widget and a label into one mega widget. To create - widgets inside a LabelFrame widget, one creates the new widgets relative to the - :attr:`frame` subwidget and manage them inside the :attr:`frame` subwidget. - -.. Python Demo of: -.. \ulink{LabelFrame}{https://tix.sourceforge.net/dist/current/demos/samples/LabFrame.tcl} - - -.. class:: Meter() - - The `Meter - `_ widget - can be used to show the progress of a background job which may take a long time - to execute. - -.. Python Demo of: -.. \ulink{Meter}{https://tix.sourceforge.net/dist/current/demos/samples/Meter.tcl} - - -.. class:: OptionMenu() - - The `OptionMenu - `_ - creates a menu button of options. - -.. Python Demo of: -.. \ulink{OptionMenu}{https://tix.sourceforge.net/dist/current/demos/samples/OptMenu.tcl} - - -.. class:: PopupMenu() - - The `PopupMenu - `_ - widget can be used as a replacement of the ``tk_popup`` command. The advantage - of the :mod:`Tix` :class:`PopupMenu` widget is it requires less application code - to manipulate. - -.. Python Demo of: -.. \ulink{PopupMenu}{https://tix.sourceforge.net/dist/current/demos/samples/PopMenu.tcl} - - -.. class:: Select() - - The `Select - `_ widget - is a container of button subwidgets. It can be used to provide radio-box or - check-box style of selection options for the user. - -.. Python Demo of: -.. \ulink{Select}{https://tix.sourceforge.net/dist/current/demos/samples/Select.tcl} - - -.. class:: StdButtonBox() - - The `StdButtonBox - `_ - widget is a group of standard buttons for Motif-like dialog boxes. - -.. Python Demo of: -.. \ulink{StdButtonBox}{https://tix.sourceforge.net/dist/current/demos/samples/StdBBox.tcl} - - -File Selectors -^^^^^^^^^^^^^^ - - -.. class:: DirList() - - The `DirList - `_ - widget displays a list view of a directory, its previous directories and its - sub-directories. The user can choose one of the directories displayed in the - list or change to another directory. - -.. Python Demo of: -.. \ulink{DirList}{https://tix.sourceforge.net/dist/current/demos/samples/DirList.tcl} - - -.. class:: DirTree() - - The `DirTree - `_ - widget displays a tree view of a directory, its previous directories and its - sub-directories. The user can choose one of the directories displayed in the - list or change to another directory. - -.. Python Demo of: -.. \ulink{DirTree}{https://tix.sourceforge.net/dist/current/demos/samples/DirTree.tcl} - - -.. class:: DirSelectDialog() - - The `DirSelectDialog - `_ - widget presents the directories in the file system in a dialog window. The user - can use this dialog window to navigate through the file system to select the - desired directory. - -.. Python Demo of: -.. \ulink{DirSelectDialog}{https://tix.sourceforge.net/dist/current/demos/samples/DirDlg.tcl} - - -.. class:: DirSelectBox() - - The :class:`DirSelectBox` is similar to the standard Motif(TM) - directory-selection box. It is generally used for the user to choose a - directory. DirSelectBox stores the directories mostly recently selected into - a ComboBox widget so that they can be quickly selected again. - - -.. class:: ExFileSelectBox() - - The `ExFileSelectBox - `_ - widget is usually embedded in a tixExFileSelectDialog widget. It provides a - convenient method for the user to select files. The style of the - :class:`ExFileSelectBox` widget is very similar to the standard file dialog on - MS Windows 3.1. - -.. Python Demo of: -.. \ulink{ExFileSelectDialog}{https://tix.sourceforge.net/dist/current/demos/samples/EFileDlg.tcl} - - -.. class:: FileSelectBox() - - The `FileSelectBox - `_ - is similar to the standard Motif(TM) file-selection box. It is generally used - for the user to choose a file. FileSelectBox stores the files mostly recently - selected into a :class:`ComboBox` widget so that they can be quickly selected - again. - -.. Python Demo of: -.. \ulink{FileSelectDialog}{https://tix.sourceforge.net/dist/current/demos/samples/FileDlg.tcl} - - -.. class:: FileEntry() - - The `FileEntry - `_ - widget can be used to input a filename. The user can type in the filename - manually. Alternatively, the user can press the button widget that sits next to - the entry, which will bring up a file selection dialog. - -.. Python Demo of: -.. \ulink{FileEntry}{https://tix.sourceforge.net/dist/current/demos/samples/FileEnt.tcl} - - -Hierarchical ListBox -^^^^^^^^^^^^^^^^^^^^ - - -.. class:: HList() - - The `HList - `_ widget - can be used to display any data that have a hierarchical structure, for example, - file system directory trees. The list entries are indented and connected by - branch lines according to their places in the hierarchy. - -.. Python Demo of: -.. \ulink{HList}{https://tix.sourceforge.net/dist/current/demos/samples/HList1.tcl} - - -.. class:: CheckList() - - The `CheckList - `_ - widget displays a list of items to be selected by the user. CheckList acts - similarly to the Tk checkbutton or radiobutton widgets, except it is capable of - handling many more items than checkbuttons or radiobuttons. - -.. Python Demo of: -.. \ulink{ CheckList}{https://tix.sourceforge.net/dist/current/demos/samples/ChkList.tcl} -.. Python Demo of: -.. \ulink{ScrolledHList (1)}{https://tix.sourceforge.net/dist/current/demos/samples/SHList.tcl} -.. Python Demo of: -.. \ulink{ScrolledHList (2)}{https://tix.sourceforge.net/dist/current/demos/samples/SHList2.tcl} - - -.. class:: Tree() - - The `Tree - `_ widget - can be used to display hierarchical data in a tree form. The user can adjust the - view of the tree by opening or closing parts of the tree. - -.. Python Demo of: -.. \ulink{Tree}{https://tix.sourceforge.net/dist/current/demos/samples/Tree.tcl} -.. Python Demo of: -.. \ulink{Tree (Dynamic)}{https://tix.sourceforge.net/dist/current/demos/samples/DynTree.tcl} - - -Tabular ListBox -^^^^^^^^^^^^^^^ - - -.. class:: TList() - - The `TList - `_ widget - can be used to display data in a tabular format. The list entries of a - :class:`TList` widget are similar to the entries in the Tk listbox widget. The - main differences are (1) the :class:`TList` widget can display the list entries - in a two dimensional format and (2) you can use graphical images as well as - multiple colors and fonts for the list entries. - -.. Python Demo of: -.. \ulink{ScrolledTList (1)}{https://tix.sourceforge.net/dist/current/demos/samples/STList1.tcl} -.. Python Demo of: -.. \ulink{ScrolledTList (2)}{https://tix.sourceforge.net/dist/current/demos/samples/STList2.tcl} -.. Grid has yet to be added to Python -.. \subsubsection{Grid Widget} -.. Python Demo of: -.. \ulink{Simple Grid}{https://tix.sourceforge.net/dist/current/demos/samples/SGrid0.tcl} -.. Python Demo of: -.. \ulink{ScrolledGrid}{https://tix.sourceforge.net/dist/current/demos/samples/SGrid1.tcl} -.. Python Demo of: -.. \ulink{Editable Grid}{https://tix.sourceforge.net/dist/current/demos/samples/EditGrid.tcl} - - -Manager Widgets -^^^^^^^^^^^^^^^ - - -.. class:: PanedWindow() - - The `PanedWindow - `_ - widget allows the user to interactively manipulate the sizes of several panes. - The panes can be arranged either vertically or horizontally. The user changes - the sizes of the panes by dragging the resize handle between two panes. - -.. Python Demo of: -.. \ulink{PanedWindow}{https://tix.sourceforge.net/dist/current/demos/samples/PanedWin.tcl} - - -.. class:: ListNoteBook() - - The `ListNoteBook - `_ - widget is very similar to the :class:`TixNoteBook` widget: it can be used to - display many windows in a limited space using a notebook metaphor. The notebook - is divided into a stack of pages (windows). At one time only one of these pages - can be shown. The user can navigate through these pages by choosing the name of - the desired page in the :attr:`hlist` subwidget. - -.. Python Demo of: -.. \ulink{ListNoteBook}{https://tix.sourceforge.net/dist/current/demos/samples/ListNBK.tcl} - - -.. class:: NoteBook() - - The `NoteBook - `_ - widget can be used to display many windows in a limited space using a notebook - metaphor. The notebook is divided into a stack of pages. At one time only one of - these pages can be shown. The user can navigate through these pages by choosing - the visual "tabs" at the top of the NoteBook widget. - -.. Python Demo of: -.. \ulink{NoteBook}{https://tix.sourceforge.net/dist/current/demos/samples/NoteBook.tcl} - -.. \subsubsection{Scrolled Widgets} -.. Python Demo of: -.. \ulink{ScrolledListBox}{https://tix.sourceforge.net/dist/current/demos/samples/SListBox.tcl} -.. Python Demo of: -.. \ulink{ScrolledText}{https://tix.sourceforge.net/dist/current/demos/samples/SText.tcl} -.. Python Demo of: -.. \ulink{ScrolledWindow}{https://tix.sourceforge.net/dist/current/demos/samples/SWindow.tcl} -.. Python Demo of: -.. \ulink{Canvas Object View}{https://tix.sourceforge.net/dist/current/demos/samples/CObjView.tcl} - - -Image Types -^^^^^^^^^^^ - -The :mod:`tkinter.tix` module adds: - -* `pixmap `_ - capabilities to all :mod:`tkinter.tix` and :mod:`tkinter` widgets to create - color images from XPM files. - - .. Python Demo of: - .. \ulink{XPM Image In Button}{https://tix.sourceforge.net/dist/current/demos/samples/Xpm.tcl} - .. Python Demo of: - .. \ulink{XPM Image In Menu}{https://tix.sourceforge.net/dist/current/demos/samples/Xpm1.tcl} - -* `Compound - `_ image - types can be used to create images that consists of multiple horizontal lines; - each line is composed of a series of items (texts, bitmaps, images or spaces) - arranged from left to right. For example, a compound image can be used to - display a bitmap and a text string simultaneously in a Tk :class:`Button` - widget. - - .. Python Demo of: - .. \ulink{Compound Image In Buttons}{https://tix.sourceforge.net/dist/current/demos/samples/CmpImg.tcl} - .. Python Demo of: - .. \ulink{Compound Image In NoteBook}{https://tix.sourceforge.net/dist/current/demos/samples/CmpImg2.tcl} - .. Python Demo of: - .. \ulink{Compound Image Notebook Color Tabs}{https://tix.sourceforge.net/dist/current/demos/samples/CmpImg4.tcl} - .. Python Demo of: - .. \ulink{Compound Image Icons}{https://tix.sourceforge.net/dist/current/demos/samples/CmpImg3.tcl} - - -Miscellaneous Widgets -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: InputOnly() - - The `InputOnly - `_ - widgets are to accept inputs from the user, which can be done with the ``bind`` - command (Unix only). - - -Form Geometry Manager -^^^^^^^^^^^^^^^^^^^^^ - -In addition, :mod:`tkinter.tix` augments :mod:`tkinter` by providing: - - -.. class:: Form() - - The `Form - `_ geometry - manager based on attachment rules for all Tk widgets. - - -Tix Commands ------------- - - -.. class:: tixCommand() - - The `tix commands - `_ provide - access to miscellaneous elements of :mod:`Tix`'s internal state and the - :mod:`Tix` application context. Most of the information manipulated by these - methods pertains to the application as a whole, or to a screen or display, - rather than to a particular window. - - To view the current settings, the common usage is:: - - from tkinter import tix - root = tix.Tk() - print(root.tix_configure()) - - -.. method:: tixCommand.tix_configure(cnf=None, **kw) - - Query or modify the configuration options of the Tix application context. If no - option is specified, returns a dictionary all of the available options. If - option is specified with no value, then the method returns a list describing the - one named option (this list will be identical to the corresponding sublist of - the value returned if no option is specified). If one or more option-value - pairs are specified, then the method modifies the given option(s) to have the - given value(s); in this case the method returns an empty string. Option may be - any of the configuration options. - - -.. method:: tixCommand.tix_cget(option) - - Returns the current value of the configuration option given by *option*. Option - may be any of the configuration options. - - -.. method:: tixCommand.tix_getbitmap(name) - - Locates a bitmap file of the name ``name.xpm`` or ``name`` in one of the bitmap - directories (see the :meth:`tix_addbitmapdir` method). By using - :meth:`tix_getbitmap`, you can avoid hard coding the pathnames of the bitmap - files in your application. When successful, it returns the complete pathname of - the bitmap file, prefixed with the character ``@``. The returned value can be - used to configure the ``bitmap`` option of the Tk and Tix widgets. - - -.. method:: tixCommand.tix_addbitmapdir(directory) - - Tix maintains a list of directories under which the :meth:`tix_getimage` and - :meth:`tix_getbitmap` methods will search for image files. The standard bitmap - directory is :file:`$TIX_LIBRARY/bitmaps`. The :meth:`tix_addbitmapdir` method - adds *directory* into this list. By using this method, the image files of an - applications can also be located using the :meth:`tix_getimage` or - :meth:`tix_getbitmap` method. - - -.. method:: tixCommand.tix_filedialog([dlgclass]) - - Returns the file selection dialog that may be shared among different calls from - this application. This method will create a file selection dialog widget when - it is called the first time. This dialog will be returned by all subsequent - calls to :meth:`tix_filedialog`. An optional dlgclass parameter can be passed - as a string to specified what type of file selection dialog widget is desired. - Possible options are ``tix``, ``FileSelectDialog`` or ``tixExFileSelectDialog``. - - -.. method:: tixCommand.tix_getimage(self, name) - - Locates an image file of the name :file:`name.xpm`, :file:`name.xbm` or - :file:`name.ppm` in one of the bitmap directories (see the - :meth:`tix_addbitmapdir` method above). If more than one file with the same name - (but different extensions) exist, then the image type is chosen according to the - depth of the X display: xbm images are chosen on monochrome displays and color - images are chosen on color displays. By using :meth:`tix_getimage`, you can - avoid hard coding the pathnames of the image files in your application. When - successful, this method returns the name of the newly created image, which can - be used to configure the ``image`` option of the Tk and Tix widgets. - - -.. method:: tixCommand.tix_option_get(name) - - Gets the options maintained by the Tix scheme mechanism. - - -.. method:: tixCommand.tix_resetoptions(newScheme, newFontSet[, newScmPrio]) - - Resets the scheme and fontset of the Tix application to *newScheme* and - *newFontSet*, respectively. This affects only those widgets created after this - call. Therefore, it is best to call the resetoptions method before the creation - of any widgets in a Tix application. - - The optional parameter *newScmPrio* can be given to reset the priority level of - the Tk options set by the Tix schemes. - - Because of the way Tk handles the X option database, after Tix has been has - imported and inited, it is not possible to reset the color schemes and font sets - using the :meth:`tix_config` method. Instead, the :meth:`tix_resetoptions` - method must be used. diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 4ff2b2159c3622..6e01ec7b291255 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -102,35 +102,35 @@ themed widgets and is not supposed to be directly instantiated. Standard Options ^^^^^^^^^^^^^^^^ -All the :mod:`ttk` Widgets accepts the following options: - - .. tabularcolumns:: |l|L| - - +-----------+--------------------------------------------------------------+ - | Option | Description | - +===========+==============================================================+ - | class | Specifies the window class. The class is used when querying | - | | the option database for the window's other options, to | - | | determine the default bindtags for the window, and to select | - | | the widget's default layout and style. This option is | - | | read-only, and may only be specified when the window is | - | | created. | - +-----------+--------------------------------------------------------------+ - | cursor | Specifies the mouse cursor to be used for the widget. If set | - | | to the empty string (the default), the cursor is inherited | - | | for the parent widget. | - +-----------+--------------------------------------------------------------+ - | takefocus | Determines whether the window accepts the focus during | - | | keyboard traversal. 0, 1 or an empty string is returned. | - | | If 0 is returned, it means that the window should be skipped | - | | entirely during keyboard traversal. If 1, it means that the | - | | window should receive the input focus as long as it is | - | | viewable. And an empty string means that the traversal | - | | scripts make the decision about whether or not to focus | - | | on the window. | - +-----------+--------------------------------------------------------------+ - | style | May be used to specify a custom widget style. | - +-----------+--------------------------------------------------------------+ +All the :mod:`ttk` Widgets accept the following options: + +.. tabularcolumns:: |l|L| + ++-----------+--------------------------------------------------------------+ +| Option | Description | ++===========+==============================================================+ +| class | Specifies the window class. The class is used when querying | +| | the option database for the window's other options, to | +| | determine the default bindtags for the window, and to select | +| | the widget's default layout and style. This option is | +| | read-only, and may only be specified when the window is | +| | created. | ++-----------+--------------------------------------------------------------+ +| cursor | Specifies the mouse cursor to be used for the widget. If set | +| | to the empty string (the default), the cursor is inherited | +| | for the parent widget. | ++-----------+--------------------------------------------------------------+ +| takefocus | Determines whether the window accepts the focus during | +| | keyboard traversal. 0, 1 or an empty string is returned. | +| | If 0 is returned, it means that the window should be skipped | +| | entirely during keyboard traversal. If 1, it means that the | +| | window should receive the input focus as long as it is | +| | viewable. And an empty string means that the traversal | +| | scripts make the decision about whether or not to focus | +| | on the window. | ++-----------+--------------------------------------------------------------+ +| style | May be used to specify a custom widget style. | ++-----------+--------------------------------------------------------------+ Scrollable Widget Options @@ -139,24 +139,24 @@ Scrollable Widget Options The following options are supported by widgets that are controlled by a scrollbar. - .. tabularcolumns:: |l|L| - - +----------------+---------------------------------------------------------+ - | Option | Description | - +================+=========================================================+ - | xscrollcommand | Used to communicate with horizontal scrollbars. | - | | | - | | When the view in the widget's window change, the widget | - | | will generate a Tcl command based on the scrollcommand. | - | | | - | | Usually this option consists of the method | - | | :meth:`Scrollbar.set` of some scrollbar. This will cause| - | | the scrollbar to be updated whenever the view in the | - | | window changes. | - +----------------+---------------------------------------------------------+ - | yscrollcommand | Used to communicate with vertical scrollbars. | - | | For some more information, see above. | - +----------------+---------------------------------------------------------+ +.. tabularcolumns:: |l|L| + ++----------------+---------------------------------------------------------+ +| Option | Description | ++================+=========================================================+ +| xscrollcommand | Used to communicate with horizontal scrollbars. | +| | | +| | When the view in the widget's window change, the widget | +| | will generate a Tcl command based on the scrollcommand. | +| | | +| | Usually this option consists of the method | +| | :meth:`Scrollbar.set` of some scrollbar. This will cause| +| | the scrollbar to be updated whenever the view in the | +| | window changes. | ++----------------+---------------------------------------------------------+ +| yscrollcommand | Used to communicate with vertical scrollbars. | +| | For some more information, see above. | ++----------------+---------------------------------------------------------+ Label Options @@ -165,93 +165,93 @@ Label Options The following options are supported by labels, buttons and other button-like widgets. - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +--------------+-----------------------------------------------------------+ - | Option | Description | - +==============+===========================================================+ - | text | Specifies a text string to be displayed inside the widget.| - +--------------+-----------------------------------------------------------+ - | textvariable | Specifies a name whose value will be used in place of the | - | | text option resource. | - +--------------+-----------------------------------------------------------+ - | underline | If set, specifies the index (0-based) of a character to | - | | underline in the text string. The underline character is | - | | used for mnemonic activation. | - +--------------+-----------------------------------------------------------+ - | image | Specifies an image to display. This is a list of 1 or more| - | | elements. The first element is the default image name. The| - | | rest of the list if a sequence of statespec/value pairs as| - | | defined by :meth:`Style.map`, specifying different images | - | | to use when the widget is in a particular state or a | - | | combination of states. All images in the list should have | - | | the same size. | - +--------------+-----------------------------------------------------------+ - | compound | Specifies how to display the image relative to the text, | - | | in the case both text and images options are present. | - | | Valid values are: | - | | | - | | * text: display text only | - | | * image: display image only | - | | * top, bottom, left, right: display image above, below, | - | | left of, or right of the text, respectively. | - | | * none: the default. display the image if present, | - | | otherwise the text. | - +--------------+-----------------------------------------------------------+ - | width | If greater than zero, specifies how much space, in | - | | character widths, to allocate for the text label, if less | - | | than zero, specifies a minimum width. If zero or | - | | unspecified, the natural width of the text label is used. | - +--------------+-----------------------------------------------------------+ +.. tabularcolumns:: |l|p{0.7\linewidth}| + ++--------------+-----------------------------------------------------------+ +| Option | Description | ++==============+===========================================================+ +| text | Specifies a text string to be displayed inside the widget.| ++--------------+-----------------------------------------------------------+ +| textvariable | Specifies a name whose value will be used in place of the | +| | text option resource. | ++--------------+-----------------------------------------------------------+ +| underline | If set, specifies the index (0-based) of a character to | +| | underline in the text string. The underline character is | +| | used for mnemonic activation. | ++--------------+-----------------------------------------------------------+ +| image | Specifies an image to display. This is a list of 1 or more| +| | elements. The first element is the default image name. The| +| | rest of the list if a sequence of statespec/value pairs as| +| | defined by :meth:`Style.map`, specifying different images | +| | to use when the widget is in a particular state or a | +| | combination of states. All images in the list should have | +| | the same size. | ++--------------+-----------------------------------------------------------+ +| compound | Specifies how to display the image relative to the text, | +| | in the case both text and images options are present. | +| | Valid values are: | +| | | +| | * text: display text only | +| | * image: display image only | +| | * top, bottom, left, right: display image above, below, | +| | left of, or right of the text, respectively. | +| | * none: the default. display the image if present, | +| | otherwise the text. | ++--------------+-----------------------------------------------------------+ +| width | If greater than zero, specifies how much space, in | +| | character widths, to allocate for the text label, if less | +| | than zero, specifies a minimum width. If zero or | +| | unspecified, the natural width of the text label is used. | ++--------------+-----------------------------------------------------------+ Compatibility Options ^^^^^^^^^^^^^^^^^^^^^ - .. tabularcolumns:: |l|L| +.. tabularcolumns:: |l|L| - +--------+----------------------------------------------------------------+ - | Option | Description | - +========+================================================================+ - | state | May be set to "normal" or "disabled" to control the "disabled" | - | | state bit. This is a write-only option: setting it changes the | - | | widget state, but the :meth:`Widget.state` method does not | - | | affect this option. | - +--------+----------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Option | Description | ++========+================================================================+ +| state | May be set to "normal" or "disabled" to control the "disabled" | +| | state bit. This is a write-only option: setting it changes the | +| | widget state, but the :meth:`Widget.state` method does not | +| | affect this option. | ++--------+----------------------------------------------------------------+ Widget States ^^^^^^^^^^^^^ The widget state is a bitmap of independent state flags. - .. tabularcolumns:: |l|L| - - +------------+-------------------------------------------------------------+ - | Flag | Description | - +============+=============================================================+ - | active | The mouse cursor is over the widget and pressing a mouse | - | | button will cause some action to occur | - +------------+-------------------------------------------------------------+ - | disabled | Widget is disabled under program control | - +------------+-------------------------------------------------------------+ - | focus | Widget has keyboard focus | - +------------+-------------------------------------------------------------+ - | pressed | Widget is being pressed | - +------------+-------------------------------------------------------------+ - | selected | "On", "true", or "current" for things like Checkbuttons and | - | | radiobuttons | - +------------+-------------------------------------------------------------+ - | background | Windows and Mac have a notion of an "active" or foreground | - | | window. The *background* state is set for widgets in a | - | | background window, and cleared for those in the foreground | - | | window | - +------------+-------------------------------------------------------------+ - | readonly | Widget should not allow user modification | - +------------+-------------------------------------------------------------+ - | alternate | A widget-specific alternate display format | - +------------+-------------------------------------------------------------+ - | invalid | The widget's value is invalid | - +------------+-------------------------------------------------------------+ +.. tabularcolumns:: |l|L| + ++------------+-------------------------------------------------------------+ +| Flag | Description | ++============+=============================================================+ +| active | The mouse cursor is over the widget and pressing a mouse | +| | button will cause some action to occur | ++------------+-------------------------------------------------------------+ +| disabled | Widget is disabled under program control | ++------------+-------------------------------------------------------------+ +| focus | Widget has keyboard focus | ++------------+-------------------------------------------------------------+ +| pressed | Widget is being pressed | ++------------+-------------------------------------------------------------+ +| selected | "On", "true", or "current" for things like Checkbuttons and | +| | radiobuttons | ++------------+-------------------------------------------------------------+ +| background | Windows and Mac have a notion of an "active" or foreground | +| | window. The *background* state is set for widgets in a | +| | background window, and cleared for those in the foreground | +| | window | ++------------+-------------------------------------------------------------+ +| readonly | Widget should not allow user modification | ++------------+-------------------------------------------------------------+ +| alternate | A widget-specific alternate display format | ++------------+-------------------------------------------------------------+ +| invalid | The widget's value is invalid | ++------------+-------------------------------------------------------------+ A state specification is a sequence of state names, optionally prefixed with an exclamation point indicating that the bit is off. @@ -311,43 +311,43 @@ Options This widget accepts the following specific options: - .. tabularcolumns:: |l|L| - - +-----------------+--------------------------------------------------------+ - | Option | Description | - +=================+========================================================+ - | exportselection | Boolean value. If set, the widget selection is linked | - | | to the Window Manager selection (which can be returned | - | | by invoking Misc.selection_get, for example). | - +-----------------+--------------------------------------------------------+ - | justify | Specifies how the text is aligned within the widget. | - | | One of "left", "center", or "right". | - +-----------------+--------------------------------------------------------+ - | height | Specifies the height of the pop-down listbox, in rows. | - +-----------------+--------------------------------------------------------+ - | postcommand | A script (possibly registered with Misc.register) that | - | | is called immediately before displaying the values. It | - | | may specify which values to display. | - +-----------------+--------------------------------------------------------+ - | state | One of "normal", "readonly", or "disabled". In the | - | | "readonly" state, the value may not be edited directly,| - | | and the user can only selection of the values from the | - | | dropdown list. In the "normal" state, the text field is| - | | directly editable. In the "disabled" state, no | - | | interaction is possible. | - +-----------------+--------------------------------------------------------+ - | textvariable | Specifies a name whose value is linked to the widget | - | | value. Whenever the value associated with that name | - | | changes, the widget value is updated, and vice versa. | - | | See :class:`tkinter.StringVar`. | - +-----------------+--------------------------------------------------------+ - | values | Specifies the list of values to display in the | - | | drop-down listbox. | - +-----------------+--------------------------------------------------------+ - | width | Specifies an integer value indicating the desired width| - | | of the entry window, in average-size characters of the | - | | widget's font. | - +-----------------+--------------------------------------------------------+ +.. tabularcolumns:: |l|L| + ++-----------------+--------------------------------------------------------+ +| Option | Description | ++=================+========================================================+ +| exportselection | Boolean value. If set, the widget selection is linked | +| | to the Window Manager selection (which can be returned | +| | by invoking Misc.selection_get, for example). | ++-----------------+--------------------------------------------------------+ +| justify | Specifies how the text is aligned within the widget. | +| | One of "left", "center", or "right". | ++-----------------+--------------------------------------------------------+ +| height | Specifies the height of the pop-down listbox, in rows. | ++-----------------+--------------------------------------------------------+ +| postcommand | A script (possibly registered with Misc.register) that | +| | is called immediately before displaying the values. It | +| | may specify which values to display. | ++-----------------+--------------------------------------------------------+ +| state | One of "normal", "readonly", or "disabled". In the | +| | "readonly" state, the value may not be edited directly,| +| | and the user can only selection of the values from the | +| | dropdown list. In the "normal" state, the text field is| +| | directly editable. In the "disabled" state, no | +| | interaction is possible. | ++-----------------+--------------------------------------------------------+ +| textvariable | Specifies a name whose value is linked to the widget | +| | value. Whenever the value associated with that name | +| | changes, the widget value is updated, and vice versa. | +| | See :class:`tkinter.StringVar`. | ++-----------------+--------------------------------------------------------+ +| values | Specifies the list of values to display in the | +| | drop-down listbox. | ++-----------------+--------------------------------------------------------+ +| width | Specifies an integer value indicating the desired width| +| | of the entry window, in average-size characters of the | +| | widget's font. | ++-----------------+--------------------------------------------------------+ Virtual events @@ -397,7 +397,7 @@ Options This widget accepts the following specific options: - .. tabularcolumns:: |l|L| +.. tabularcolumns:: |l|L| +----------------------+------------------------------------------------------+ | Option | Description | @@ -473,25 +473,25 @@ Options This widget accepts the following specific options: - .. tabularcolumns:: |l|L| - - +---------+----------------------------------------------------------------+ - | Option | Description | - +=========+================================================================+ - | height | If present and greater than zero, specifies the desired height | - | | of the pane area (not including internal padding or tabs). | - | | Otherwise, the maximum height of all panes is used. | - +---------+----------------------------------------------------------------+ - | padding | Specifies the amount of extra space to add around the outside | - | | of the notebook. The padding is a list up to four length | - | | specifications left top right bottom. If fewer than four | - | | elements are specified, bottom defaults to top, right defaults | - | | to left, and top defaults to left. | - +---------+----------------------------------------------------------------+ - | width | If present and greater than zero, specified the desired width | - | | of the pane area (not including internal padding). Otherwise, | - | | the maximum width of all panes is used. | - +---------+----------------------------------------------------------------+ +.. tabularcolumns:: |l|L| + ++---------+----------------------------------------------------------------+ +| Option | Description | ++=========+================================================================+ +| height | If present and greater than zero, specifies the desired height | +| | of the pane area (not including internal padding or tabs). | +| | Otherwise, the maximum height of all panes is used. | ++---------+----------------------------------------------------------------+ +| padding | Specifies the amount of extra space to add around the outside | +| | of the notebook. The padding is a list up to four length | +| | specifications left top right bottom. If fewer than four | +| | elements are specified, bottom defaults to top, right defaults | +| | to left, and top defaults to left. | ++---------+----------------------------------------------------------------+ +| width | If present and greater than zero, specified the desired width | +| | of the pane area (not including internal padding). Otherwise, | +| | the maximum width of all panes is used. | ++---------+----------------------------------------------------------------+ Tab Options @@ -499,39 +499,39 @@ Tab Options There are also specific options for tabs: - .. tabularcolumns:: |l|L| - - +-----------+--------------------------------------------------------------+ - | Option | Description | - +===========+==============================================================+ - | state | Either "normal", "disabled" or "hidden". If "disabled", then | - | | the tab is not selectable. If "hidden", then the tab is not | - | | shown. | - +-----------+--------------------------------------------------------------+ - | sticky | Specifies how the child window is positioned within the pane | - | | area. Value is a string containing zero or more of the | - | | characters "n", "s", "e" or "w". Each letter refers to a | - | | side (north, south, east or west) that the child window will | - | | stick to, as per the :meth:`grid` geometry manager. | - +-----------+--------------------------------------------------------------+ - | padding | Specifies the amount of extra space to add between the | - | | notebook and this pane. Syntax is the same as for the option | - | | padding used by this widget. | - +-----------+--------------------------------------------------------------+ - | text | Specifies a text to be displayed in the tab. | - +-----------+--------------------------------------------------------------+ - | image | Specifies an image to display in the tab. See the option | - | | image described in :class:`Widget`. | - +-----------+--------------------------------------------------------------+ - | compound | Specifies how to display the image relative to the text, in | - | | the case both options text and image are present. See | - | | `Label Options`_ for legal values. | - +-----------+--------------------------------------------------------------+ - | underline | Specifies the index (0-based) of a character to underline in | - | | the text string. The underlined character is used for | - | | mnemonic activation if :meth:`Notebook.enable_traversal` is | - | | called. | - +-----------+--------------------------------------------------------------+ +.. tabularcolumns:: |l|L| + ++-----------+--------------------------------------------------------------+ +| Option | Description | ++===========+==============================================================+ +| state | Either "normal", "disabled" or "hidden". If "disabled", then | +| | the tab is not selectable. If "hidden", then the tab is not | +| | shown. | ++-----------+--------------------------------------------------------------+ +| sticky | Specifies how the child window is positioned within the pane | +| | area. Value is a string containing zero or more of the | +| | characters "n", "s", "e" or "w". Each letter refers to a | +| | side (north, south, east or west) that the child window will | +| | stick to, as per the :meth:`grid` geometry manager. | ++-----------+--------------------------------------------------------------+ +| padding | Specifies the amount of extra space to add between the | +| | notebook and this pane. Syntax is the same as for the option | +| | padding used by this widget. | ++-----------+--------------------------------------------------------------+ +| text | Specifies a text to be displayed in the tab. | ++-----------+--------------------------------------------------------------+ +| image | Specifies an image to display in the tab. See the option | +| | image described in :class:`Widget`. | ++-----------+--------------------------------------------------------------+ +| compound | Specifies how to display the image relative to the text, in | +| | the case both options text and image are present. See | +| | `Label Options`_ for legal values. | ++-----------+--------------------------------------------------------------+ +| underline | Specifies the index (0-based) of a character to underline in | +| | the text string. The underlined character is used for | +| | mnemonic activation if :meth:`Notebook.enable_traversal` is | +| | called. | ++-----------+--------------------------------------------------------------+ Tab Identifiers @@ -663,36 +663,36 @@ Options This widget accepts the following specific options: - .. tabularcolumns:: |l|L| - - +----------+---------------------------------------------------------------+ - | Option | Description | - +==========+===============================================================+ - | orient | One of "horizontal" or "vertical". Specifies the orientation | - | | of the progress bar. | - +----------+---------------------------------------------------------------+ - | length | Specifies the length of the long axis of the progress bar | - | | (width if horizontal, height if vertical). | - +----------+---------------------------------------------------------------+ - | mode | One of "determinate" or "indeterminate". | - +----------+---------------------------------------------------------------+ - | maximum | A number specifying the maximum value. Defaults to 100. | - +----------+---------------------------------------------------------------+ - | value | The current value of the progress bar. In "determinate" mode, | - | | this represents the amount of work completed. In | - | | "indeterminate" mode, it is interpreted as modulo *maximum*; | - | | that is, the progress bar completes one "cycle" when its value| - | | increases by *maximum*. | - +----------+---------------------------------------------------------------+ - | variable | A name which is linked to the option value. If specified, the | - | | value of the progress bar is automatically set to the value of| - | | this name whenever the latter is modified. | - +----------+---------------------------------------------------------------+ - | phase | Read-only option. The widget periodically increments the value| - | | of this option whenever its value is greater than 0 and, in | - | | determinate mode, less than maximum. This option may be used | - | | by the current theme to provide additional animation effects. | - +----------+---------------------------------------------------------------+ +.. tabularcolumns:: |l|L| + ++----------+---------------------------------------------------------------+ +| Option | Description | ++==========+===============================================================+ +| orient | One of "horizontal" or "vertical". Specifies the orientation | +| | of the progress bar. | ++----------+---------------------------------------------------------------+ +| length | Specifies the length of the long axis of the progress bar | +| | (width if horizontal, height if vertical). | ++----------+---------------------------------------------------------------+ +| mode | One of "determinate" or "indeterminate". | ++----------+---------------------------------------------------------------+ +| maximum | A number specifying the maximum value. Defaults to 100. | ++----------+---------------------------------------------------------------+ +| value | The current value of the progress bar. In "determinate" mode, | +| | this represents the amount of work completed. In | +| | "indeterminate" mode, it is interpreted as modulo *maximum*; | +| | that is, the progress bar completes one "cycle" when its value| +| | increases by *maximum*. | ++----------+---------------------------------------------------------------+ +| variable | A name which is linked to the option value. If specified, the | +| | value of the progress bar is automatically set to the value of| +| | this name whenever the latter is modified. | ++----------+---------------------------------------------------------------+ +| phase | Read-only option. The widget periodically increments the value| +| | of this option whenever its value is greater than 0 and, in | +| | determinate mode, less than maximum. This option may be used | +| | by the current theme to provide additional animation effects. | ++----------+---------------------------------------------------------------+ ttk.Progressbar @@ -734,14 +734,14 @@ Options This widget accepts the following specific option: - .. tabularcolumns:: |l|L| +.. tabularcolumns:: |l|L| - +--------+----------------------------------------------------------------+ - | Option | Description | - +========+================================================================+ - | orient | One of "horizontal" or "vertical". Specifies the orientation of| - | | the separator. | - +--------+----------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Option | Description | ++========+================================================================+ +| orient | One of "horizontal" or "vertical". Specifies the orientation of| +| | the separator. | ++--------+----------------------------------------------------------------+ Sizegrip @@ -802,49 +802,49 @@ Options This widget accepts the following specific options: - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +----------------+--------------------------------------------------------+ - | Option | Description | - +================+========================================================+ - | columns | A list of column identifiers, specifying the number of | - | | columns and their names. | - +----------------+--------------------------------------------------------+ - | displaycolumns | A list of column identifiers (either symbolic or | - | | integer indices) specifying which data columns are | - | | displayed and the order in which they appear, or the | - | | string "#all". | - +----------------+--------------------------------------------------------+ - | height | Specifies the number of rows which should be visible. | - | | Note: the requested width is determined from the sum | - | | of the column widths. | - +----------------+--------------------------------------------------------+ - | padding | Specifies the internal padding for the widget. The | - | | padding is a list of up to four length specifications. | - +----------------+--------------------------------------------------------+ - | selectmode | Controls how the built-in class bindings manage the | - | | selection. One of "extended", "browse" or "none". | - | | If set to "extended" (the default), multiple items may | - | | be selected. If "browse", only a single item will be | - | | selected at a time. If "none", the selection will not | - | | be changed. | - | | | - | | Note that the application code and tag bindings can set| - | | the selection however they wish, regardless of the | - | | value of this option. | - +----------------+--------------------------------------------------------+ - | show | A list containing zero or more of the following values,| - | | specifying which elements of the tree to display. | - | | | - | | * tree: display tree labels in column #0. | - | | * headings: display the heading row. | - | | | - | | The default is "tree headings", i.e., show all | - | | elements. | - | | | - | | **Note**: Column #0 always refers to the tree column, | - | | even if show="tree" is not specified. | - +----------------+--------------------------------------------------------+ +.. tabularcolumns:: |l|p{0.7\linewidth}| + ++----------------+--------------------------------------------------------+ +| Option | Description | ++================+========================================================+ +| columns | A list of column identifiers, specifying the number of | +| | columns and their names. | ++----------------+--------------------------------------------------------+ +| displaycolumns | A list of column identifiers (either symbolic or | +| | integer indices) specifying which data columns are | +| | displayed and the order in which they appear, or the | +| | string "#all". | ++----------------+--------------------------------------------------------+ +| height | Specifies the number of rows which should be visible. | +| | Note: the requested width is determined from the sum | +| | of the column widths. | ++----------------+--------------------------------------------------------+ +| padding | Specifies the internal padding for the widget. The | +| | padding is a list of up to four length specifications. | ++----------------+--------------------------------------------------------+ +| selectmode | Controls how the built-in class bindings manage the | +| | selection. One of "extended", "browse" or "none". | +| | If set to "extended" (the default), multiple items may | +| | be selected. If "browse", only a single item will be | +| | selected at a time. If "none", the selection will not | +| | be changed. | +| | | +| | Note that the application code and tag bindings can set| +| | the selection however they wish, regardless of the | +| | value of this option. | ++----------------+--------------------------------------------------------+ +| show | A list containing zero or more of the following values,| +| | specifying which elements of the tree to display. | +| | | +| | * tree: display tree labels in column #0. | +| | * headings: display the heading row. | +| | | +| | The default is "tree headings", i.e., show all | +| | elements. | +| | | +| | **Note**: Column #0 always refers to the tree column, | +| | even if show="tree" is not specified. | ++----------------+--------------------------------------------------------+ Item Options @@ -853,27 +853,27 @@ Item Options The following item options may be specified for items in the insert and item widget commands. - .. tabularcolumns:: |l|L| - - +--------+---------------------------------------------------------------+ - | Option | Description | - +========+===============================================================+ - | text | The textual label to display for the item. | - +--------+---------------------------------------------------------------+ - | image | A Tk Image, displayed to the left of the label. | - +--------+---------------------------------------------------------------+ - | values | The list of values associated with the item. | - | | | - | | Each item should have the same number of values as the widget | - | | option columns. If there are fewer values than columns, the | - | | remaining values are assumed empty. If there are more values | - | | than columns, the extra values are ignored. | - +--------+---------------------------------------------------------------+ - | open | ``True``/``False`` value indicating whether the item's | - | | children should be displayed or hidden. | - +--------+---------------------------------------------------------------+ - | tags | A list of tags associated with this item. | - +--------+---------------------------------------------------------------+ +.. tabularcolumns:: |l|L| + ++--------+---------------------------------------------------------------+ +| Option | Description | ++========+===============================================================+ +| text | The textual label to display for the item. | ++--------+---------------------------------------------------------------+ +| image | A Tk Image, displayed to the left of the label. | ++--------+---------------------------------------------------------------+ +| values | The list of values associated with the item. | +| | | +| | Each item should have the same number of values as the widget | +| | option columns. If there are fewer values than columns, the | +| | remaining values are assumed empty. If there are more values | +| | than columns, the extra values are ignored. | ++--------+---------------------------------------------------------------+ +| open | ``True``/``False`` value indicating whether the item's | +| | children should be displayed or hidden. | ++--------+---------------------------------------------------------------+ +| tags | A list of tags associated with this item. | ++--------+---------------------------------------------------------------+ Tag Options @@ -881,20 +881,20 @@ Tag Options The following options may be specified on tags: - .. tabularcolumns:: |l|L| +.. tabularcolumns:: |l|L| - +------------+-----------------------------------------------------------+ - | Option | Description | - +============+===========================================================+ - | foreground | Specifies the text foreground color. | - +------------+-----------------------------------------------------------+ - | background | Specifies the cell or item background color. | - +------------+-----------------------------------------------------------+ - | font | Specifies the font to use when drawing text. | - +------------+-----------------------------------------------------------+ - | image | Specifies the item image, in case the item's image option | - | | is empty. | - +------------+-----------------------------------------------------------+ ++------------+-----------------------------------------------------------+ +| Option | Description | ++============+===========================================================+ +| foreground | Specifies the text foreground color. | ++------------+-----------------------------------------------------------+ +| background | Specifies the cell or item background color. | ++------------+-----------------------------------------------------------+ +| font | Specifies the font to use when drawing text. | ++------------+-----------------------------------------------------------+ +| image | Specifies the item image, in case the item's image option | +| | is empty. | ++------------+-----------------------------------------------------------+ Column Identifiers @@ -926,19 +926,19 @@ Virtual Events The Treeview widget generates the following virtual events. - .. tabularcolumns:: |l|L| +.. tabularcolumns:: |l|L| - +--------------------+--------------------------------------------------+ - | Event | Description | - +====================+==================================================+ - | <> | Generated whenever the selection changes. | - +--------------------+--------------------------------------------------+ - | <> | Generated just before settings the focus item to | - | | open=True. | - +--------------------+--------------------------------------------------+ - | <> | Generated just after setting the focus item to | - | | open=False. | - +--------------------+--------------------------------------------------+ ++--------------------+--------------------------------------------------+ +| Event | Description | ++====================+==================================================+ +| <> | Generated whenever the selection changes. | ++--------------------+--------------------------------------------------+ +| <> | Generated just before settings the focus item to | +| | open=True. | ++--------------------+--------------------------------------------------+ +| <> | Generated just after setting the focus item to | +| | open=False. | ++--------------------+--------------------------------------------------+ The :meth:`Treeview.focus` and :meth:`Treeview.selection` methods can be used to determine the affected item or items. @@ -986,19 +986,19 @@ ttk.Treeview The valid options/values are: - * id + id Returns the column name. This is a read-only option. - * anchor: One of the standard Tk anchor values. + anchor: One of the standard Tk anchor values. Specifies how the text in this column should be aligned with respect to the cell. - * minwidth: width + minwidth: width The minimum width of the column in pixels. The treeview widget will not make the column any smaller than specified by this option when the widget is resized or the user drags a column. - * stretch: ``True``/``False`` + stretch: ``True``/``False`` Specifies whether the column's width should be adjusted when the widget is resized. - * width: width + width: width The width of the column in pixels. To configure the tree column, call this with column = "#0" @@ -1041,14 +1041,14 @@ ttk.Treeview The valid options/values are: - * text: text + text: text The text to display in the column heading. - * image: imageName + image: imageName Specifies an image to display to the right of the column heading. - * anchor: anchor + anchor: anchor Specifies how the heading text should be aligned. One of the standard Tk anchor values. - * command: callback + command: callback A callback to be invoked when the heading label is pressed. To configure the tree column heading, call this with column = "#0". @@ -1391,32 +1391,42 @@ option. If you don't know the class name of a widget, use the method .. method:: element_create(elementname, etype, *args, **kw) Create a new element in the current theme, of the given *etype* which is - expected to be either "image", "from" or "vsapi". The latter is only - available in Tk 8.6a for Windows XP and Vista and is not described here. + expected to be either "image", "from" or "vsapi". + The latter is only available in Tk 8.6 on Windows. If "image" is used, *args* should contain the default image name followed by statespec/value pairs (this is the imagespec), and *kw* may have the following options: - * border=padding - padding is a list of up to four integers, specifying the left, top, - right, and bottom borders, respectively. + border=padding + padding is a list of up to four integers, specifying the left, top, + right, and bottom borders, respectively. - * height=height - Specifies a minimum height for the element. If less than zero, the - base image's height is used as a default. + height=height + Specifies a minimum height for the element. If less than zero, the + base image's height is used as a default. - * padding=padding - Specifies the element's interior padding. Defaults to border's value - if not specified. + padding=padding + Specifies the element's interior padding. Defaults to border's value + if not specified. - * sticky=spec - Specifies how the image is placed within the final parcel. spec - contains zero or more characters "n", "s", "w", or "e". + sticky=spec + Specifies how the image is placed within the final parcel. spec + contains zero or more characters "n", "s", "w", or "e". - * width=width - Specifies a minimum width for the element. If less than zero, the - base image's width is used as a default. + width=width + Specifies a minimum width for the element. If less than zero, the + base image's width is used as a default. + + Example:: + + img1 = tkinter.PhotoImage(master=root, file='button.png') + img1 = tkinter.PhotoImage(master=root, file='button-pressed.png') + img1 = tkinter.PhotoImage(master=root, file='button-active.png') + style = ttk.Style(root) + style.element_create('Button.button', 'image', + img1, ('pressed', img2), ('active', img3), + border=(2, 4), sticky='we') If "from" is used as the value of *etype*, :meth:`element_create` will clone an existing @@ -1425,6 +1435,68 @@ option. If you don't know the class name of a widget, use the method If this element to clone from is not specified, an empty element will be used. *kw* is discarded. + Example:: + + style = ttk.Style(root) + style.element_create('plain.background', 'from', 'default') + + If "vsapi" is used as the value of *etype*, :meth:`element_create` + will create a new element in the current theme whose visual appearance + is drawn using the Microsoft Visual Styles API which is responsible + for the themed styles on Windows XP and Vista. + *args* is expected to contain the Visual Styles class and part as + given in the Microsoft documentation followed by an optional sequence + of tuples of ttk states and the corresponding Visual Styles API state + value. + *kw* may have the following options: + + padding=padding + Specify the element's interior padding. + *padding* is a list of up to four integers specifying the left, + top, right and bottom padding quantities respectively. + If fewer than four elements are specified, bottom defaults to top, + right defaults to left, and top defaults to left. + In other words, a list of three numbers specify the left, vertical, + and right padding; a list of two numbers specify the horizontal + and the vertical padding; a single number specifies the same + padding all the way around the widget. + This option may not be mixed with any other options. + + margins=padding + Specifies the elements exterior padding. + *padding* is a list of up to four integers specifying the left, top, + right and bottom padding quantities respectively. + This option may not be mixed with any other options. + + width=width + Specifies the width for the element. + If this option is set then the Visual Styles API will not be queried + for the recommended size or the part. + If this option is set then *height* should also be set. + The *width* and *height* options cannot be mixed with the *padding* + or *margins* options. + + height=height + Specifies the height of the element. + See the comments for *width*. + + Example:: + + style = ttk.Style(root) + style.element_create('pin', 'vsapi', 'EXPLORERBAR', 3, [ + ('pressed', '!selected', 3), + ('active', '!selected', 2), + ('pressed', 'selected', 6), + ('active', 'selected', 5), + ('selected', 4), + ('', 1)]) + style.layout('Explorer.Pin', + [('Explorer.Pin.pin', {'sticky': 'news'})]) + pin = ttk.Checkbutton(style='Explorer.Pin') + pin.pack(expand=True, fill='both') + + .. versionchanged:: 3.13 + Added support of the "vsapi" element factory. .. method:: element_names() @@ -1504,22 +1576,22 @@ uses a simplified version of the pack geometry manager: given an initial cavity, each element is allocated a parcel. Valid options/values are: - * side: whichside - Specifies which side of the cavity to place the element; one of - top, right, bottom or left. If omitted, the element occupies the - entire cavity. +side: whichside + Specifies which side of the cavity to place the element; one of + top, right, bottom or left. If omitted, the element occupies the + entire cavity. - * sticky: nswe - Specifies where the element is placed inside its allocated parcel. +sticky: nswe + Specifies where the element is placed inside its allocated parcel. - * unit: 0 or 1 - If set to 1, causes the element and all of its descendants to be treated as - a single element for the purposes of :meth:`Widget.identify` et al. It's - used for things like scrollbar thumbs with grips. +unit: 0 or 1 + If set to 1, causes the element and all of its descendants to be treated as + a single element for the purposes of :meth:`Widget.identify` et al. It's + used for things like scrollbar thumbs with grips. - * children: [sublayout... ] - Specifies a list of elements to place inside the element. Each - element is a tuple (or other sequence type) where the first item is - the layout name, and the other is a `Layout`_. +children: [sublayout... ] + Specifies a list of elements to place inside the element. Each + element is a tuple (or other sequence type) where the first item is + the layout name, and the other is a `Layout`_. .. _Layout: `Layouts`_ diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc index 3b345099bf54b5..39df2927a0b7f2 100644 --- a/Doc/library/token-list.inc +++ b/Doc/library/token-list.inc @@ -207,10 +207,6 @@ .. data:: OP -.. data:: AWAIT - -.. data:: ASYNC - .. data:: TYPE_IGNORE .. data:: TYPE_COMMENT @@ -223,6 +219,10 @@ .. data:: FSTRING_END +.. data:: COMMENT + +.. data:: NL + .. data:: ERRORTOKEN .. data:: N_TOKENS diff --git a/Doc/library/token.rst b/Doc/library/token.rst index a1aceba96ce030..e6dc37d7ad852c 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -50,11 +50,13 @@ The following token type values aren't used by the C tokenizer but are needed fo the :mod:`tokenize` module. .. data:: COMMENT + :noindex: Token value used to indicate a comment. .. data:: NL + :noindex: Token value used to indicate a non-terminating newline. The :data:`NEWLINE` token indicates the end of a logical line of Python code; @@ -78,17 +80,21 @@ the :mod:`tokenize` module. .. versionchanged:: 3.5 - Added :data:`AWAIT` and :data:`ASYNC` tokens. + Added :data:`!AWAIT` and :data:`!ASYNC` tokens. .. versionchanged:: 3.7 Added :data:`COMMENT`, :data:`NL` and :data:`ENCODING` tokens. .. versionchanged:: 3.7 - Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are + Removed :data:`!AWAIT` and :data:`!ASYNC` tokens. "async" and "await" are now tokenized as :data:`NAME` tokens. .. versionchanged:: 3.8 Added :data:`TYPE_COMMENT`, :data:`TYPE_IGNORE`, :data:`COLONEQUAL`. - Added :data:`AWAIT` and :data:`ASYNC` tokens back (they're needed + Added :data:`!AWAIT` and :data:`!ASYNC` tokens back (they're needed to support parsing older Python versions for :func:`ast.parse` with ``feature_version`` set to 6 or lower). + +.. versionchanged:: 3.13 + Removed :data:`!AWAIT` and :data:`!ASYNC` tokens again. + diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 11f569df2e7cde..92bdb052267a68 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -22,6 +22,15 @@ the generic :data:`~token.OP` token type. The exact type can be determined by checking the ``exact_type`` property on the :term:`named tuple` returned from :func:`tokenize.tokenize`. + +.. warning:: + + Note that the functions in this module are only designed to parse + syntactically valid Python code (code that does not raise when parsed + using :func:`ast.parse`). The behavior of the functions in this module is + **undefined** when providing invalid Python code and it can change at any + point. + Tokenizing Input ---------------- @@ -139,11 +148,6 @@ function it uses to do this is available: 2, 3 -Note that unclosed single-quoted strings do not cause an error to be -raised. They are tokenized as :data:`~token.ERRORTOKEN`, followed by the -tokenization of their contents. - - .. _tokenize-cli: Command-Line Usage @@ -162,11 +166,11 @@ The following options are accepted: .. program:: tokenize -.. cmdoption:: -h, --help +.. option:: -h, --help show this help message and exit -.. cmdoption:: -e, --exact +.. option:: -e, --exact display token names using the exact type diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 40cf198f1287d7..8854905e192b45 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -34,11 +34,11 @@ all Python modules imported during the execution into the current directory. .. program:: trace -.. cmdoption:: --help +.. option:: --help Display usage and exit. -.. cmdoption:: --version +.. option:: --version Display the version of the module and exit. @@ -56,28 +56,28 @@ the :option:`--trace <-t>` and :option:`--count <-c>` options. When .. program:: trace -.. cmdoption:: -c, --count +.. option:: -c, --count Produce a set of annotated listing files upon program completion that shows how many times each statement was executed. See also :option:`--coverdir <-C>`, :option:`--file <-f>` and :option:`--no-report <-R>` below. -.. cmdoption:: -t, --trace +.. option:: -t, --trace Display lines as they are executed. -.. cmdoption:: -l, --listfuncs +.. option:: -l, --listfuncs Display the functions executed by running the program. -.. cmdoption:: -r, --report +.. option:: -r, --report Produce an annotated list from an earlier program run that used the :option:`--count <-c>` and :option:`--file <-f>` option. This does not execute any code. -.. cmdoption:: -T, --trackcalls +.. option:: -T, --trackcalls Display the calling relationships exposed by running the program. @@ -86,33 +86,33 @@ Modifiers .. program:: trace -.. cmdoption:: -f, --file= +.. option:: -f, --file= Name of a file to accumulate counts over several tracing runs. Should be used with the :option:`--count <-c>` option. -.. cmdoption:: -C, --coverdir= +.. option:: -C, --coverdir= Directory where the report files go. The coverage report for ``package.module`` is written to file :file:`{dir}/{package}/{module}.cover`. -.. cmdoption:: -m, --missing +.. option:: -m, --missing When generating annotated listings, mark lines which were not executed with ``>>>>>>``. -.. cmdoption:: -s, --summary +.. option:: -s, --summary When using :option:`--count <-c>` or :option:`--report <-r>`, write a brief summary to stdout for each file processed. -.. cmdoption:: -R, --no-report +.. option:: -R, --no-report Do not generate annotated listings. This is useful if you intend to make several runs with :option:`--count <-c>`, and then produce a single set of annotated listings at the end. -.. cmdoption:: -g, --timing +.. option:: -g, --timing Prefix each line with the time since the program started. Only used while tracing. @@ -124,12 +124,12 @@ These options may be repeated multiple times. .. program:: trace -.. cmdoption:: --ignore-module= +.. option:: --ignore-module= Ignore each of the given module names and its submodules (if it is a package). The argument can be a list of names separated by a comma. -.. cmdoption:: --ignore-dir= +.. option:: --ignore-dir= Ignore all modules and packages in the named directory and subdirectories. The argument can be a list of directories separated by :data:`os.pathsep`. @@ -187,7 +187,8 @@ Programmatic Interface Merge in data from another :class:`CoverageResults` object. - .. method:: write_results(show_missing=True, summary=False, coverdir=None) + .. method:: write_results(show_missing=True, summary=False, coverdir=None,\ + *, ignore_missing_files=False) Write coverage results. Set *show_missing* to show lines that had no hits. Set *summary* to include in the output the coverage summary per @@ -195,6 +196,13 @@ Programmatic Interface result files will be output. If ``None``, the results for each source file are placed in its directory. + If *ignore_missing_files* is ``True``, coverage counts for files that no + longer exist are silently ignored. Otherwise, a missing file will + raise a :exc:`FileNotFoundError`. + + .. versionchanged:: 3.13 + Added *ignore_missing_files* parameter. + A simple example demonstrating the use of the programmatic interface:: import sys diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 5c0e261b90763c..2d5ea19b2cb892 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -86,10 +86,9 @@ The module defines the following functions: .. function:: print_last(limit=None, file=None, chain=True) - This is a shorthand for ``print_exception(sys.last_type, sys.last_value, - sys.last_traceback, limit, file, chain)``. In general it will work only - after an exception has reached an interactive prompt (see - :data:`sys.last_type`). + This is a shorthand for ``print_exception(sys.last_exc, limit, file, + chain)``. In general it will work only after an exception has reached + an interactive prompt (see :data:`sys.last_exc`). .. function:: print_stack(f=None, limit=None, file=None) @@ -136,24 +135,34 @@ The module defines the following functions: text line is not ``None``. -.. function:: format_exception_only(exc, /[, value]) +.. function:: format_exception_only(exc, /[, value], *, show_group=False) Format the exception part of a traceback using an exception value such as given by ``sys.last_value``. The return value is a list of strings, each - ending in a newline. Normally, the list contains a single string; however, - for :exc:`SyntaxError` exceptions, it contains several lines that (when - printed) display detailed information about where the syntax error occurred. - The message indicating which exception occurred is the always last string in - the list. + ending in a newline. The list contains the exception's message, which is + normally a single string; however, for :exc:`SyntaxError` exceptions, it + contains several lines that (when printed) display detailed information + about where the syntax error occurred. Following the message, the list + contains the exception's :attr:`notes `. Since Python 3.10, instead of passing *value*, an exception object can be passed as the first argument. If *value* is provided, the first argument is ignored in order to provide backwards compatibility. + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. + .. versionchanged:: 3.10 The *etype* parameter has been renamed to *exc* and is now positional-only. + .. versionchanged:: 3.11 + The returned list now includes any notes attached to the exception. + + .. versionchanged:: 3.13 + *show_group* parameter was added. + .. function:: format_exception(exc, /[, value, tb], limit=None, chain=True) @@ -218,7 +227,7 @@ The module also defines the following classes: :class:`TracebackException` objects are created from actual exceptions to capture data for later printing in a lightweight fashion. -.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False) +.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) Capture an exception for later rendering. *limit*, *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` class. @@ -230,6 +239,18 @@ capture data for later printing in a lightweight fashion. Note that when locals are captured, they are also shown in the traceback. + *max_group_width* and *max_group_depth* control the formatting of exception + groups (see :exc:`BaseExceptionGroup`). The depth refers to the nesting + level of the group, and the width refers to the size of a single exception + group's exceptions array. The formatted output is truncated when either + limit is exceeded. + + .. versionchanged:: 3.10 + Added the *compact* parameter. + + .. versionchanged:: 3.11 + Added the *max_group_width* and *max_group_depth* parameters. + .. attribute:: __cause__ A :class:`TracebackException` of the original ``__cause__``. @@ -238,6 +259,14 @@ capture data for later printing in a lightweight fashion. A :class:`TracebackException` of the original ``__context__``. + .. attribute:: exceptions + + If ``self`` represents an :exc:`ExceptionGroup`, this field holds a list of + :class:`TracebackException` instances representing the nested exceptions. + Otherwise it is ``None``. + + .. versionadded:: 3.11 + .. attribute:: __suppress_context__ The ``__suppress_context__`` value from the original exception. @@ -258,6 +287,14 @@ capture data for later printing in a lightweight fashion. The class of the original traceback. + .. deprecated:: 3.13 + + .. attribute:: exc_type_str + + String display of the class of the original exception. + + .. versionadded:: 3.13 + .. attribute:: filename For syntax errors - the file name where the error occurred. @@ -266,6 +303,13 @@ capture data for later printing in a lightweight fashion. For syntax errors - the line number where the error occurred. + .. attribute:: end_lineno + + For syntax errors - the end line number where the error occurred. + Can be ``None`` if not present. + + .. versionadded:: 3.10 + .. attribute:: text For syntax errors - the text where the error occurred. @@ -274,6 +318,13 @@ capture data for later printing in a lightweight fashion. For syntax errors - the offset into the text where the error occurred. + .. attribute:: end_offset + + For syntax errors - the end offset into the text where the error occurred. + Can be ``None`` if not present. + + .. versionadded:: 3.10 + .. attribute:: msg For syntax errors - the compiler error message. @@ -303,25 +354,27 @@ capture data for later printing in a lightweight fashion. some containing internal newlines. :func:`~traceback.print_exception` is a wrapper around this method which just prints the lines to a file. - The message indicating which exception occurred is always the last - string in the output. - - .. method:: format_exception_only() + .. method:: format_exception_only(*, show_group=False) Format the exception part of the traceback. The return value is a generator of strings, each ending in a newline. - Normally, the generator emits a single string; however, for - :exc:`SyntaxError` exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. + When *show_group* is ``False``, the generator emits the exception's + message followed by its notes (if it has any). The exception message + is normally a single string; however, for :exc:`SyntaxError` exceptions, + it consists of several lines that (when printed) display detailed + information about where the syntax error occurred. - The message indicating which exception occurred is always the last - string in the output. + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. - .. versionchanged:: 3.10 - Added the *compact* parameter. + .. versionchanged:: 3.11 + The exception's notes are now included in the output. + + .. versionchanged:: 3.13 + Added the *show_group* parameter. :class:`StackSummary` Objects @@ -470,27 +523,32 @@ The output for the example would look similar to this: *** print_tb: File "", line 10, in lumberjack() + ~~~~~~~~~~^^ *** print_exception: Traceback (most recent call last): File "", line 10, in lumberjack() + ~~~~~~~~~~^^ File "", line 4, in lumberjack bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ IndexError: tuple index out of range *** print_exc: Traceback (most recent call last): File "", line 10, in lumberjack() + ~~~~~~~~~~^^ File "", line 4, in lumberjack bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ IndexError: tuple index out of range *** format_exc, first and last line: Traceback (most recent call last): IndexError: tuple index out of range *** format_exception: ['Traceback (most recent call last):\n', - ' File "", line 10, in \n lumberjack()\n', - ' File "", line 4, in lumberjack\n bright_side_of_life()\n', + ' File "", line 10, in \n lumberjack()\n ~~~~~~~~~~^^\n', + ' File "", line 4, in lumberjack\n bright_side_of_life()\n ~~~~~~~~~~~~~~~~~~~^^\n', ' File "", line 7, in bright_side_of_life\n return tuple()[0]\n ~~~~~~~^^^\n', 'IndexError: tuple index out of range\n'] *** extract_tb: @@ -498,8 +556,8 @@ The output for the example would look similar to this: , line 4 in lumberjack>, , line 7 in bright_side_of_life>] *** format_tb: - [' File "", line 10, in \n lumberjack()\n', - ' File "", line 4, in lumberjack\n bright_side_of_life()\n', + [' File "", line 10, in \n lumberjack()\n ~~~~~~~~~~^^\n', + ' File "", line 4, in lumberjack\n bright_side_of_life()\n ~~~~~~~~~~~~~~~~~~~^^\n', ' File "", line 7, in bright_side_of_life\n return tuple()[0]\n ~~~~~~~^^^\n'] *** tb_lineno: 10 diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index b30bc3c7ac42e9..20ba7d7e0a45b3 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -15,23 +15,49 @@ The :mod:`tty` module defines functions for putting the tty into cbreak and raw modes. +.. availability:: Unix. + Because it requires the :mod:`termios` module, it will work only on Unix. The :mod:`tty` module defines the following functions: +.. function:: cfmakeraw(mode) + + Convert the tty attribute list *mode*, which is a list like the one returned + by :func:`termios.tcgetattr`, to that of a tty in raw mode. + + .. versionadded:: 3.12 + + +.. function:: cfmakecbreak(mode) + + Convert the tty attribute list *mode*, which is a list like the one returned + by :func:`termios.tcgetattr`, to that of a tty in cbreak mode. + + .. versionadded:: 3.12 + + .. function:: setraw(fd, when=termios.TCSAFLUSH) Change the mode of the file descriptor *fd* to raw. If *when* is omitted, it defaults to :const:`termios.TCSAFLUSH`, and is passed to - :func:`termios.tcsetattr`. + :func:`termios.tcsetattr`. The return value of :func:`termios.tcgetattr` + is saved before setting *fd* to raw mode; this value is returned. + + .. versionchanged:: 3.12 + The return value is now the original tty attributes, instead of None. .. function:: setcbreak(fd, when=termios.TCSAFLUSH) Change the mode of file descriptor *fd* to cbreak. If *when* is omitted, it defaults to :const:`termios.TCSAFLUSH`, and is passed to - :func:`termios.tcsetattr`. + :func:`termios.tcsetattr`. The return value of :func:`termios.tcgetattr` + is saved before setting *fd* to cbreak mode; this value is returned. + + .. versionchanged:: 3.12 + The return value is now the original tty attributes, instead of None. .. seealso:: diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 10138f4f406f85..88b1f09eb3c8b7 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -19,9 +19,14 @@ Introduction ============ -Turtle graphics is a popular way for introducing programming to kids. It was -part of the original Logo programming language developed by Wally Feurzeig, -Seymour Papert and Cynthia Solomon in 1967. +Turtle graphics is an implementation of `the popular geometric drawing tools +introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon +in 1967. + + +Get started +=========== Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the @@ -36,67 +41,261 @@ direction it is facing, drawing a line as it moves. Give it the command .. image:: turtle-star.* :align: center - .. literalinclude:: ../includes/turtle-star.py +In Python, turtle graphics provides a representation of a physical "turtle" +(a little robot with a pen) that draws on a sheet of paper on the floor. + +It's an effective and well-proven way for learners to encounter +programming concepts and interaction with software, as it provides instant, +visible feedback. It also provides convenient access to graphical output +in general. + +Turtle drawing was originally created as an educational tool, to be used by +teachers in the classroom. For the programmer who needs to produce some +graphical output it can be a way to do that without the overhead of +introducing more complex or external libraries into their work. + + +.. _turtle-tutorial: + +Tutorial +======== + +New users should start here. In this tutorial we'll explore some of the +basics of turtle drawing. + + +Starting a turtle environment +----------------------------- + +In a Python shell, import all the objects of the ``turtle`` module:: + + from turtle import * + +If you run into a ``No module named '_tkinter'`` error, you'll have to +install the :mod:`Tk interface package ` on your system. + + +Basic drawing +------------- + +Send the turtle forward 100 steps:: + + forward(100) + +You should see (most likely, in a new window on your display) a line +drawn by the turtle, heading East. Change the direction of the turtle, +so that it turns 120 degrees left (anti-clockwise):: + + left(120) + +Let's continue by drawing a triangle:: + + forward(100) + left(120) + forward(100) + +Notice how the turtle, represented by an arrow, points in different +directions as you steer it. + +Experiment with those commands, and also with ``backward()`` and +``right()``. + + +Pen control +~~~~~~~~~~~ + +Try changing the color - for example, ``color('blue')`` - and +width of the line - for example, ``width(3)`` - and then drawing again. + +You can also move the turtle around without drawing, by lifting up the pen: +``up()`` before moving. To start drawing again, use ``down()``. + + +The turtle's position +~~~~~~~~~~~~~~~~~~~~~ -By combining together these and similar commands, intricate shapes and pictures -can easily be drawn. +Send your turtle back to its starting-point (useful if it has disappeared +off-screen):: -The :mod:`turtle` module is an extended reimplementation of the same-named -module from the Python standard distribution up to version Python 2.5. + home() -It tries to keep the merits of the old turtle module and to be (nearly) 100% -compatible with it. This means in the first place to enable the learning -programmer to use all the commands, classes and methods interactively when using -the module from within IDLE run with the ``-n`` switch. +The home position is at the center of the turtle's screen. If you ever need to +know them, get the turtle's x-y co-ordinates with:: -The turtle module provides turtle graphics primitives, in both object-oriented -and procedure-oriented ways. Because it uses :mod:`tkinter` for the underlying -graphics, it needs a version of Python installed with Tk support. + pos() -The object-oriented interface uses essentially two+two classes: +Home is at ``(0, 0)``. -1. The :class:`TurtleScreen` class defines graphics windows as a playground for - the drawing turtles. Its constructor needs a :class:`tkinter.Canvas` or a - :class:`ScrolledCanvas` as argument. It should be used when :mod:`turtle` is - used as part of some application. +And after a while, it will probably help to clear the window so we can start +anew:: - The function :func:`Screen` returns a singleton object of a - :class:`TurtleScreen` subclass. This function should be used when - :mod:`turtle` is used as a standalone tool for doing graphics. - As a singleton object, inheriting from its class is not possible. + clearscreen() + + +Making algorithmic patterns +--------------------------- + +Using loops, it's possible to build up geometric patterns:: + + for steps in range(100): + for c in ('blue', 'red', 'green'): + color(c) + forward(steps) + right(30) + + +\ - which of course, are limited only by the imagination! + +Let's draw the star shape at the top of this page. We want red lines, +filled in with yellow:: + + color('red') + fillcolor('yellow') + +Just as ``up()`` and ``down()`` determine whether lines will be drawn, +filling can be turned on and off:: + + begin_fill() + +Next we'll create a loop:: + + while True: + forward(200) + left(170) + if abs(pos()) < 1: + break + +``abs(pos()) < 1`` is a good way to know when the turtle is back at its +home position. + +Finally, complete the filling:: + + end_fill() + +(Note that filling only actually takes place when you give the +``end_fill()`` command.) + + +.. _turtle-how-to: + +How to... +========= + +This section covers some typical turtle use-cases and approaches. + + +Get started as quickly as possible +---------------------------------- + +One of the joys of turtle graphics is the immediate, visual feedback that's +available from simple commands - it's an excellent way to introduce children +to programming ideas, with a minimum of overhead (not just children, of +course). + +The turtle module makes this possible by exposing all its basic functionality +as functions, available with ``from turtle import *``. The :ref:`turtle +graphics tutorial ` covers this approach. + +It's worth noting that many of the turtle commands also have even more terse +equivalents, such as ``fd()`` for :func:`forward`. These are especially +useful when working with learners for whom typing is not a skill. + +.. _note: + + You'll need to have the :mod:`Tk interface package ` installed on + your system for turtle graphics to work. Be warned that this is not + always straightforward, so check this in advance if you're planning to + use turtle graphics with a learner. + + +Use the ``turtle`` module namespace +----------------------------------- - All methods of TurtleScreen/Screen also exist as functions, i.e. as part of - the procedure-oriented interface. +Using ``from turtle import *`` is convenient - but be warned that it imports a +rather large collection of objects, and if you're doing anything but turtle +graphics you run the risk of a name conflict (this becomes even more an issue +if you're using turtle graphics in a script where other modules might be +imported). -2. :class:`RawTurtle` (alias: :class:`RawPen`) defines Turtle objects which draw - on a :class:`TurtleScreen`. Its constructor needs a Canvas, ScrolledCanvas - or TurtleScreen as argument, so the RawTurtle objects know where to draw. +The solution is to use ``import turtle`` - ``fd()`` becomes +``turtle.fd()``, ``width()`` becomes ``turtle.width()`` and so on. (If typing +"turtle" over and over again becomes tedious, use for example ``import turtle +as t`` instead.) - Derived from RawTurtle is the subclass :class:`Turtle` (alias: :class:`Pen`), - which draws on "the" :class:`Screen` instance which is automatically - created, if not already present. - All methods of RawTurtle/Turtle also exist as functions, i.e. part of the - procedure-oriented interface. +Use turtle graphics in a script +------------------------------- -The procedural interface provides functions which are derived from the methods -of the classes :class:`Screen` and :class:`Turtle`. They have the same names as -the corresponding methods. A screen object is automatically created whenever a -function derived from a Screen method is called. An (unnamed) turtle object is -automatically created whenever any of the functions derived from a Turtle method -is called. +It's recommended to use the ``turtle`` module namespace as described +immediately above, for example:: -To use multiple turtles on a screen one has to use the object-oriented interface. + import turtle as t + from random import random + + for i in range(100): + steps = int(random() * 100) + angle = int(random() * 360) + t.right(angle) + t.fd(steps) + +Another step is also required though - as soon as the script ends, Python +will also close the turtle's window. Add:: + + t.mainloop() + +to the end of the script. The script will now wait to be dismissed and +will not exit until it is terminated, for example by closing the turtle +graphics window. + + +Use object-oriented turtle graphics +----------------------------------- + +.. seealso:: :ref:`Explanation of the object-oriented interface ` + +Other than for very basic introductory purposes, or for trying things out +as quickly as possible, it's more usual and much more powerful to use the +object-oriented approach to turtle graphics. For example, this allows +multiple turtles on screen at once. + +In this approach, the various turtle commands are methods of objects (mostly of +``Turtle`` objects). You *can* use the object-oriented approach in the shell, +but it would be more typical in a Python script. + +The example above then becomes:: + + from turtle import Turtle + from random import random + + t = Turtle() + for i in range(100): + steps = int(random() * 100) + angle = int(random() * 360) + t.right(angle) + t.fd(steps) + + t.screen.mainloop() + +Note the last line. ``t.screen`` is an instance of the :class:`Screen` +that a Turtle instance exists on; it's created automatically along with +the turtle. + +The turtle's screen can be customised, for example:: + + t.screen.title('Object-oriented turtle demo') + t.screen.bgcolor("orange") + + +Turtle graphics reference +========================= .. note:: + In the following documentation the argument list for functions is given. Methods, of course, have the additional first argument *self* which is omitted here. -Overview of available Turtle and Screen methods -================================================= - Turtle methods -------------- @@ -166,7 +365,6 @@ Turtle state | :func:`resizemode` | :func:`shapesize` | :func:`turtlesize` | :func:`shearfactor` - | :func:`settiltangle` | :func:`tiltangle` | :func:`tilt` | :func:`shapetransform` @@ -359,18 +557,18 @@ Turtle motion .. doctest:: :skipif: _tkinter is None - >>> tp = turtle.pos() - >>> tp - (0.00,0.00) - >>> turtle.setpos(60,30) - >>> turtle.pos() - (60.00,30.00) - >>> turtle.setpos((20,80)) - >>> turtle.pos() - (20.00,80.00) - >>> turtle.setpos(tp) - >>> turtle.pos() - (0.00,0.00) + >>> tp = turtle.pos() + >>> tp + (0.00,0.00) + >>> turtle.setpos(60,30) + >>> turtle.pos() + (60.00,30.00) + >>> turtle.setpos((20,80)) + >>> turtle.pos() + (20.00,80.00) + >>> turtle.setpos(tp) + >>> turtle.pos() + (0.00,0.00) .. function:: teleport(x, y=None, *, fill_gap=False) @@ -395,18 +593,18 @@ Turtle motion .. doctest:: :skipif: _tkinter is None - >>> tp = turtle.pos() - >>> tp - (0.00,0.00) - >>> turtle.teleport(60) - >>> turtle.pos() - (60.00,0.00) - >>> turtle.teleport(y=10) - >>> turtle.pos() - (60.00,10.00) - >>> turtle.teleport(20, 30) - >>> turtle.pos() - (20.00,30.00) + >>> tp = turtle.pos() + >>> tp + (0.00,0.00) + >>> turtle.teleport(60) + >>> turtle.pos() + (60.00,0.00) + >>> turtle.teleport(y=10) + >>> turtle.pos() + (60.00,10.00) + >>> turtle.teleport(20, 30) + >>> turtle.pos() + (20.00,30.00) .. versionadded: 3.12 @@ -950,23 +1148,23 @@ Color control .. doctest:: :skipif: _tkinter is None - >>> colormode() - 1.0 - >>> turtle.pencolor() - 'red' - >>> turtle.pencolor("brown") - >>> turtle.pencolor() - 'brown' - >>> tup = (0.2, 0.8, 0.55) - >>> turtle.pencolor(tup) - >>> turtle.pencolor() - (0.2, 0.8, 0.5490196078431373) - >>> colormode(255) - >>> turtle.pencolor() - (51.0, 204.0, 140.0) - >>> turtle.pencolor('#32c18f') - >>> turtle.pencolor() - (50.0, 193.0, 143.0) + >>> colormode() + 1.0 + >>> turtle.pencolor() + 'red' + >>> turtle.pencolor("brown") + >>> turtle.pencolor() + 'brown' + >>> tup = (0.2, 0.8, 0.55) + >>> turtle.pencolor(tup) + >>> turtle.pencolor() + (0.2, 0.8, 0.5490196078431373) + >>> colormode(255) + >>> turtle.pencolor() + (51.0, 204.0, 140.0) + >>> turtle.pencolor('#32c18f') + >>> turtle.pencolor() + (50.0, 193.0, 143.0) .. function:: fillcolor(*args) @@ -999,17 +1197,17 @@ Color control .. doctest:: :skipif: _tkinter is None - >>> turtle.fillcolor("violet") - >>> turtle.fillcolor() - 'violet' - >>> turtle.pencolor() - (50.0, 193.0, 143.0) - >>> turtle.fillcolor((50, 193, 143)) # Integers, not floats - >>> turtle.fillcolor() - (50.0, 193.0, 143.0) - >>> turtle.fillcolor('#ffffff') - >>> turtle.fillcolor() - (255.0, 255.0, 255.0) + >>> turtle.fillcolor("violet") + >>> turtle.fillcolor() + 'violet' + >>> turtle.pencolor() + (50.0, 193.0, 143.0) + >>> turtle.fillcolor((50, 193, 143)) # Integers, not floats + >>> turtle.fillcolor() + (50.0, 193.0, 143.0) + >>> turtle.fillcolor('#ffffff') + >>> turtle.fillcolor() + (255.0, 255.0, 255.0) .. function:: color(*args) @@ -1038,12 +1236,12 @@ Color control .. doctest:: :skipif: _tkinter is None - >>> turtle.color("red", "green") - >>> turtle.color() - ('red', 'green') - >>> color("#285078", "#a0c8f0") - >>> color() - ((40.0, 80.0, 120.0), (160.0, 200.0, 240.0)) + >>> turtle.color("red", "green") + >>> turtle.color() + ('red', 'green') + >>> color("#285078", "#a0c8f0") + >>> color() + ((40.0, 80.0, 120.0), (160.0, 200.0, 240.0)) See also: Screen method :func:`colormode`. @@ -1065,11 +1263,11 @@ Filling .. doctest:: :skipif: _tkinter is None - >>> turtle.begin_fill() - >>> if turtle.filling(): - ... turtle.pensize(5) - ... else: - ... turtle.pensize(3) + >>> turtle.begin_fill() + >>> if turtle.filling(): + ... turtle.pensize(5) + ... else: + ... turtle.pensize(3) @@ -1275,11 +1473,11 @@ Appearance .. doctest:: :skipif: _tkinter is None - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.shearfactor(0.5) - >>> turtle.shearfactor() - 0.5 + >>> turtle.shape("circle") + >>> turtle.shapesize(5,2) + >>> turtle.shearfactor(0.5) + >>> turtle.shearfactor() + 0.5 .. function:: tilt(angle) @@ -1301,28 +1499,6 @@ Appearance >>> turtle.fd(50) -.. function:: settiltangle(angle) - - :param angle: a number - - Rotate the turtleshape to point in the direction specified by *angle*, - regardless of its current tilt-angle. *Do not* change the turtle's heading - (direction of movement). - - .. doctest:: - :skipif: _tkinter is None or 'always; deprecated method' - - >>> turtle.reset() - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.settiltangle(45) - >>> turtle.fd(50) - >>> turtle.settiltangle(-45) - >>> turtle.fd(50) - - .. deprecated:: 3.1 - - .. function:: tiltangle(angle=None) :param angle: a number (optional) @@ -1648,11 +1824,11 @@ Window control ``"nopic"``, delete background image, if present. If *picname* is ``None``, return the filename of the current backgroundimage. :: - >>> screen.bgpic() - 'nopic' - >>> screen.bgpic("landscape.gif") - >>> screen.bgpic() - "landscape.gif" + >>> screen.bgpic() + 'nopic' + >>> screen.bgpic("landscape.gif") + >>> screen.bgpic() + "landscape.gif" .. function:: clear() @@ -2059,16 +2235,16 @@ Settings and special methods Return the height of the turtle window. :: - >>> screen.window_height() - 480 + >>> screen.window_height() + 480 .. function:: window_width() Return the width of the turtle window. :: - >>> screen.window_width() - 640 + >>> screen.window_width() + 640 .. _screenspecific: @@ -2139,7 +2315,7 @@ Public classes .. class:: RawTurtle(canvas) RawPen(canvas) - :param canvas: a :class:`tkinter.Canvas`, a :class:`ScrolledCanvas` or a + :param canvas: a :class:`!tkinter.Canvas`, a :class:`ScrolledCanvas` or a :class:`TurtleScreen` Create a turtle. The turtle has all methods described above as "methods of @@ -2154,7 +2330,7 @@ Public classes .. class:: TurtleScreen(cv) - :param cv: a :class:`tkinter.Canvas` + :param cv: a :class:`!tkinter.Canvas` Provides screen oriented methods like :func:`bgcolor` etc. that are described above. @@ -2224,6 +2400,41 @@ Public classes * ``a.rotate(angle)`` rotation +.. _turtle-explanation: + +Explanation +=========== + +A turtle object draws on a screen object, and there a number of key classes in +the turtle object-oriented interface that can be used to create them and relate +them to each other. + +A :class:`Turtle` instance will automatically create a :class:`Screen` +instance if one is not already present. + +``Turtle`` is a subclass of :class:`RawTurtle`, which *doesn't* automatically +create a drawing surface - a *canvas* will need to be provided or created for +it. The *canvas* can be a :class:`!tkinter.Canvas`, :class:`ScrolledCanvas` +or :class:`TurtleScreen`. + + +:class:`TurtleScreen` is the basic drawing surface for a +turtle. :class:`Screen` is a subclass of ``TurtleScreen``, and +includes :ref:`some additional methods ` for managing its +appearance (including size and title) and behaviour. ``TurtleScreen``'s +constructor needs a :class:`!tkinter.Canvas` or a +:class:`ScrolledCanvas` as an argument. + +The functional interface for turtle graphics uses the various methods of +``Turtle`` and ``TurtleScreen``/``Screen``. Behind the scenes, a screen +object is automatically created whenever a function derived from a ``Screen`` +method is called. Similarly, a turtle object is automatically created +whenever any of the functions derived from a Turtle method is called. + +To use multiple turtles on a screen, the object-oriented interface must be +used. + + Help and configuration ====================== @@ -2249,12 +2460,12 @@ facilities: in the range 0..colormode or a 3-tuple of such numbers. - >>> screen.bgcolor("orange") - >>> screen.bgcolor() - "orange" - >>> screen.bgcolor(0.5,0,0.5) - >>> screen.bgcolor() - "#800080" + >>> screen.bgcolor("orange") + >>> screen.bgcolor() + "orange" + >>> screen.bgcolor(0.5,0,0.5) + >>> screen.bgcolor() + "#800080" >>> help(Turtle.penup) Help on method penup in module turtle: @@ -2529,8 +2740,7 @@ Changes since Python 3.0 :func:`get_shapepoly` have been added. Thus the full range of regular linear transforms is now available for transforming turtle shapes. :func:`tiltangle` has been enhanced in functionality: it now can - be used to get or set the tilt angle. :func:`settiltangle` has been - deprecated. + be used to get or set the tilt angle. - The :class:`Screen` method :func:`onkeypress` has been added as a complement to :func:`onkey`. As the latter binds actions to the key release event, diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 8cbe17df16f107..22766462822af9 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -200,6 +200,8 @@ Standard names are defined for the following types: Return a copy of the code object with new values for the specified fields. + Code objects are also supported by generic function :func:`copy.replace`. + .. versionadded:: 3.8 .. data:: CellType @@ -386,7 +388,8 @@ Standard names are defined for the following types: .. data:: GetSetDescriptorType The type of objects defined in extension modules with ``PyGetSetDef``, such - as ``FrameType.f_locals`` or ``array.array.typecode``. This type is used as + as :attr:`FrameType.f_locals ` or ``array.array.typecode``. + This type is used as descriptor for object attributes; it has the same purpose as the :class:`property` type, but for classes defined in extension modules. @@ -470,6 +473,12 @@ Standard names are defined for the following types: .. versionadded:: 3.12 +.. class:: CapsuleType + + The type of :ref:`capsule objects `. + + .. versionadded:: 3.13 + Additional Utility Classes and Functions ---------------------------------------- @@ -502,6 +511,8 @@ Additional Utility Classes and Functions However, for a structured record type use :func:`~collections.namedtuple` instead. + :class:`!SimpleNamespace` objects are supported by :func:`copy.replace`. + .. versionadded:: 3.3 .. versionchanged:: 3.9 diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c300c4257f0e81..ba2845eb17ddcc 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2,6 +2,12 @@ :mod:`typing` --- Support for type hints ======================================== +.. testsetup:: * + + import typing + from dataclasses import dataclass + from typing import * + .. module:: typing :synopsis: Support for type hints (see :pep:`484`). @@ -12,15 +18,14 @@ .. note:: The Python runtime does not enforce function and variable type annotations. - They can be used by third party tools such as type checkers, IDEs, linters, - etc. + They can be used by third party tools such as :term:`type checkers `, + IDEs, linters, etc. -------------- -This module provides runtime support for type hints. The most fundamental -support consists of the types :data:`Any`, :data:`Union`, :data:`Callable`, -:class:`TypeVar`, and :class:`Generic`. For a specification, please see -:pep:`484`. For a simplified introduction to type hints, see :pep:`483`. +This module provides runtime support for type hints. For the original +specification of the typing system, see :pep:`484`. For a simplified +introduction to type hints, see :pep:`483`. The function below takes and returns a string and is annotated as follows:: @@ -41,16 +46,18 @@ For a summary of deprecated features and a deprecation timeline, please see .. seealso:: - For a quick overview of type hints, refer to - `this cheat sheet `_. + `"Typing cheat sheet" `_ + A quick overview of type hints (hosted at the mypy docs) - The "Type System Reference" section of https://mypy.readthedocs.io/ -- since - the Python typing system is standardised via PEPs, this reference should - broadly apply to most Python type checkers, although some parts may still be - specific to mypy. + "Type System Reference" section of `the mypy docs `_ + The Python typing system is standardised via PEPs, so this reference + should broadly apply to most Python type checkers. (Some parts may still + be specific to mypy.) - The documentation at https://typing.readthedocs.io/ serves as useful reference - for type system features, useful typing related tools and typing best practices. + `"Static Typing with Python" `_ + Type-checker-agnostic documentation written by the community detailing + type system features, useful typing related tools and typing best + practices. .. _relevant-peps: @@ -59,7 +66,12 @@ Relevant PEPs Since the initial introduction of type hints in :pep:`484` and :pep:`483`, a number of PEPs have modified and enhanced Python's framework for type -annotations. These include: +annotations: + +.. raw:: html + +
+ The full list of PEPs * :pep:`526`: Syntax for Variable Annotations *Introducing* syntax for annotating variables outside of function @@ -101,18 +113,27 @@ annotations. These include: * :pep:`692`: Using ``TypedDict`` for more precise ``**kwargs`` typing *Introducing* a new way of typing ``**kwargs`` with :data:`Unpack` and :data:`TypedDict` +* :pep:`695`: Type Parameter Syntax + *Introducing* builtin syntax for creating generic functions, classes, and type aliases. * :pep:`698`: Adding an override decorator to typing *Introducing* the :func:`@override` decorator +.. raw:: html + +
+
+ .. _type-aliases: Type aliases ============ -A type alias is defined by assigning the type to the alias. In this example, -``Vector`` and ``list[float]`` will be treated as interchangeable synonyms:: +A type alias is defined using the :keyword:`type` statement, which creates +an instance of :class:`TypeAliasType`. In this example, +``Vector`` and ``list[float]`` will be treated equivalently by static type +checkers:: - Vector = list[float] + type Vector = list[float] def scale(scalar: float, vector: Vector) -> Vector: return [scalar * num for num in vector] @@ -124,9 +145,9 @@ Type aliases are useful for simplifying complex type signatures. For example:: from collections.abc import Sequence - ConnectionOptions = dict[str, str] - Address = tuple[str, int] - Server = tuple[Address, ConnectionOptions] + type ConnectionOptions = dict[str, str] + type Address = tuple[str, int] + type Server = tuple[Address, ConnectionOptions] def broadcast_message(message: str, servers: Sequence[Server]) -> None: ... @@ -138,8 +159,17 @@ Type aliases are useful for simplifying complex type signatures. For example:: servers: Sequence[tuple[tuple[str, int], dict[str, str]]]) -> None: ... -Note that ``None`` as a type hint is a special case and is replaced by -``type(None)``. +The :keyword:`type` statement is new in Python 3.12. For backwards +compatibility, type aliases can also be created through simple assignment:: + + Vector = list[float] + +Or marked with :data:`TypeAlias` to make it explicit that this is a type alias, +not a normal variable assignment:: + + from typing import TypeAlias + + Vector: TypeAlias = list[float] .. _distinct: @@ -206,7 +236,7 @@ See :pep:`484` for more details. .. note:: Recall that the use of a type alias declares two types to be *equivalent* to - one another. Doing ``Alias = Original`` will make the static type checker + one another. Doing ``type Alias = Original`` will make the static type checker treat ``Alias`` as being *exactly equivalent* to ``Original`` in all cases. This is useful when you want to simplify complex type signatures. @@ -220,35 +250,85 @@ See :pep:`484` for more details. .. versionadded:: 3.5.2 .. versionchanged:: 3.10 - ``NewType`` is now a class rather than a function. There is some additional - runtime cost when calling ``NewType`` over a regular function. However, this - cost will be reduced in 3.11.0. + ``NewType`` is now a class rather than a function. As a result, there is + some additional runtime cost when calling ``NewType`` over a regular + function. +.. versionchanged:: 3.11 + The performance of calling ``NewType`` has been restored to its level in + Python 3.9. -Callable -======== +.. _annotating-callables: -Frameworks expecting callback functions of specific signatures might be -type hinted using ``Callable[[Arg1Type, Arg2Type], ReturnType]``. +Annotating callable objects +=========================== -For example:: +Functions -- or other :term:`callable` objects -- can be annotated using +:class:`collections.abc.Callable` or :data:`typing.Callable`. +``Callable[[int], str]`` signifies a function that takes a single parameter +of type :class:`int` and returns a :class:`str`. + +For example: + +.. testcode:: - from collections.abc import Callable + from collections.abc import Callable, Awaitable def feeder(get_next_item: Callable[[], str]) -> None: - # Body + ... # Body def async_query(on_success: Callable[[int], None], on_error: Callable[[int, Exception], None]) -> None: - # Body + ... # Body async def on_update(value: str) -> None: - # Body + ... # Body + callback: Callable[[str], Awaitable[None]] = on_update -It is possible to declare the return type of a callable without specifying -the call signature by substituting a literal ellipsis -for the list of arguments in the type hint: ``Callable[..., ReturnType]``. +The subscription syntax must always be used with exactly two values: the +argument list and the return type. The argument list must be a list of types, +a :class:`ParamSpec`, :data:`Concatenate`, or an ellipsis. The return type must +be a single type. + +If a literal ellipsis ``...`` is given as the argument list, it indicates that +a callable with any arbitrary parameter list would be acceptable: + +.. testcode:: + + def concat(x: str, y: str) -> str: + return x + y + + x: Callable[..., str] + x = str # OK + x = concat # Also OK + +``Callable`` cannot express complex signatures such as functions that take a +variadic number of arguments, :ref:`overloaded functions `, or +functions that have keyword-only parameters. However, these signatures can be +expressed by defining a :class:`Protocol` class with a +:meth:`~object.__call__` method: + +.. testcode:: + + from collections.abc import Iterable + from typing import Protocol + + class Combiner(Protocol): + def __call__(self, *vals: bytes, maxlen: int | None = None) -> list[bytes]: ... + + def batch_proc(data: Iterable[bytes], cb_results: Combiner) -> bytes: + for item in data: + ... + + def good_cb(*vals: bytes, maxlen: int | None = None) -> list[bytes]: + ... + def bad_cb(*vals: bytes, maxitems: int | None) -> list[bytes]: + ... + + batch_proc([], good_cb) # OK + batch_proc([], bad_cb) # Error! Argument 2 has incompatible type because of + # different name and kind in the callback Callables which take other callables as arguments may indicate that their parameter types are dependent on each other using :class:`ParamSpec`. @@ -272,28 +352,150 @@ Generics ======== Since type information about objects kept in containers cannot be statically -inferred in a generic way, abstract base classes have been extended to support -subscription to denote expected types for container elements. +inferred in a generic way, many container classes in the standard library support +subscription to denote the expected types of container elements. -:: +.. testcode:: from collections.abc import Mapping, Sequence + class Employee: ... + + # Sequence[Employee] indicates that all elements in the sequence + # must be instances of "Employee". + # Mapping[str, str] indicates that all keys and all values in the mapping + # must be strings. def notify_by_email(employees: Sequence[Employee], overrides: Mapping[str, str]) -> None: ... -Generics can be parameterized by using a factory available in typing -called :class:`TypeVar`. +Generic functions and classes can be parameterized by using +:ref:`type parameter syntax `:: -:: + from collections.abc import Sequence + + def first[T](l: Sequence[T]) -> T: # Function is generic over the TypeVar "T" + return l[0] + +Or by using the :class:`TypeVar` factory directly:: from collections.abc import Sequence from typing import TypeVar - T = TypeVar('T') # Declare type variable + U = TypeVar('U') # Declare type variable "U" - def first(l: Sequence[T]) -> T: # Generic function - return l[0] + def second(l: Sequence[U]) -> U: # Function is generic over the TypeVar "U" + return l[1] + +.. versionchanged:: 3.12 + Syntactic support for generics is new in Python 3.12. + +.. _annotating-tuples: + +Annotating tuples +================= + +For most containers in Python, the typing system assumes that all elements in +the container will be of the same type. For example:: + + from collections.abc import Mapping + + # Type checker will infer that all elements in ``x`` are meant to be ints + x: list[int] = [] + + # Type checker error: ``list`` only accepts a single type argument: + y: list[int, str] = [1, 'foo'] + + # Type checker will infer that all keys in ``z`` are meant to be strings, + # and that all values in ``z`` are meant to be either strings or ints + z: Mapping[str, str | int] = {} + +:class:`list` only accepts one type argument, so a type checker would emit an +error on the ``y`` assignment above. Similarly, +:class:`~collections.abc.Mapping` only accepts two type arguments: the first +indicates the type of the keys, and the second indicates the type of the +values. + +Unlike most other Python containers, however, it is common in idiomatic Python +code for tuples to have elements which are not all of the same type. For this +reason, tuples are special-cased in Python's typing system. :class:`tuple` +accepts *any number* of type arguments:: + + # OK: ``x`` is assigned to a tuple of length 1 where the sole element is an int + x: tuple[int] = (5,) + + # OK: ``y`` is assigned to a tuple of length 2; + # element 1 is an int, element 2 is a str + y: tuple[int, str] = (5, "foo") + + # Error: the type annotation indicates a tuple of length 1, + # but ``z`` has been assigned to a tuple of length 3 + z: tuple[int] = (1, 2, 3) + +To denote a tuple which could be of *any* length, and in which all elements are +of the same type ``T``, use ``tuple[T, ...]``. To denote an empty tuple, use +``tuple[()]``. Using plain ``tuple`` as an annotation is equivalent to using +``tuple[Any, ...]``:: + + x: tuple[int, ...] = (1, 2) + # These reassignments are OK: ``tuple[int, ...]`` indicates x can be of any length + x = (1, 2, 3) + x = () + # This reassignment is an error: all elements in ``x`` must be ints + x = ("foo", "bar") + + # ``y`` can only ever be assigned to an empty tuple + y: tuple[()] = () + + z: tuple = ("foo", "bar") + # These reassignments are OK: plain ``tuple`` is equivalent to ``tuple[Any, ...]`` + z = (1, 2, 3) + z = () + +.. _type-of-class-objects: + +The type of class objects +========================= + +A variable annotated with ``C`` may accept a value of type ``C``. In +contrast, a variable annotated with ``type[C]`` (or +:class:`typing.Type[C] `) may accept values that are classes +themselves -- specifically, it will accept the *class object* of ``C``. For +example:: + + a = 3 # Has type ``int`` + b = int # Has type ``type[int]`` + c = type(a) # Also has type ``type[int]`` + +Note that ``type[C]`` is covariant:: + + class User: ... + class ProUser(User): ... + class TeamUser(User): ... + + def make_new_user(user_class: type[User]) -> User: + # ... + return user_class() + + make_new_user(User) # OK + make_new_user(ProUser) # Also OK: ``type[ProUser]`` is a subtype of ``type[User]`` + make_new_user(TeamUser) # Still fine + make_new_user(User()) # Error: expected ``type[User]`` but got ``User`` + make_new_user(int) # Error: ``type[int]`` is not a subtype of ``type[User]`` + +The only legal parameters for :class:`type` are classes, :data:`Any`, +:ref:`type variables `, and unions of any of these types. +For example:: + + def new_non_team_user(user_class: type[BasicUser | ProUser]): ... + + new_non_team_user(BasicUser) # OK + new_non_team_user(ProUser) # OK + new_non_team_user(TeamUser) # Error: ``type[TeamUser]`` is not a subtype + # of ``type[BasicUser | ProUser]`` + new_non_team_user(User) # Also an error + +``type[Any]`` is equivalent to :class:`type`, which is the root of Python's +:ref:`metaclass hierarchy `. .. _user-defined-generics: @@ -304,12 +506,9 @@ A user-defined class can be defined as a generic class. :: - from typing import TypeVar, Generic from logging import Logger - T = TypeVar('T') - - class LoggedVar(Generic[T]): + class LoggedVar[T]: def __init__(self, value: T, name: str, logger: Logger) -> None: self.name = name self.logger = logger @@ -326,12 +525,23 @@ A user-defined class can be defined as a generic class. def log(self, message: str) -> None: self.logger.info('%s: %s', self.name, message) -``Generic[T]`` as a base class defines that the class ``LoggedVar`` takes a -single type parameter ``T`` . This also makes ``T`` valid as a type within the -class body. +This syntax indicates that the class ``LoggedVar`` is parameterised around a +single :ref:`type variable ` ``T`` . This also makes ``T`` valid as +a type within the class body. + +Generic classes implicitly inherit from :class:`Generic`. For compatibility +with Python 3.11 and lower, it is also possible to inherit explicitly from +:class:`Generic` to indicate a generic class:: + + from typing import TypeVar, Generic + + T = TypeVar('T') + + class LoggedVar(Generic[T]): + ... -The :class:`Generic` base class defines :meth:`~object.__class_getitem__` so -that ``LoggedVar[T]`` is valid as a type:: +Generic classes have :meth:`~object.__class_getitem__` methods, meaning they +can be parameterised at runtime (e.g. ``LoggedVar[int]`` below):: from collections.abc import Iterable @@ -344,11 +554,14 @@ A generic type can have any number of type variables. All varieties of from typing import TypeVar, Generic, Sequence - T = TypeVar('T', contravariant=True) - B = TypeVar('B', bound=Sequence[bytes], covariant=True) - S = TypeVar('S', int, str) + class WeirdTrio[T, B: Sequence[bytes], S: (int, str)]: + ... + + OldT = TypeVar('OldT', contravariant=True) + OldB = TypeVar('OldB', bound=Sequence[bytes], covariant=True) + OldS = TypeVar('OldS', int, str) - class WeirdTrio(Generic[T, B, S]): + class OldWeirdTrio(Generic[OldT, OldB, OldS]): ... Each type variable argument to :class:`Generic` must be distinct. @@ -357,91 +570,108 @@ This is thus invalid:: from typing import TypeVar, Generic ... + class Pair[M, M]: # SyntaxError + ... + T = TypeVar('T') class Pair(Generic[T, T]): # INVALID ... -You can use multiple inheritance with :class:`Generic`:: +Generic classes can also inherit from other classes:: from collections.abc import Sized - from typing import TypeVar, Generic - - T = TypeVar('T') - class LinkedList(Sized, Generic[T]): + class LinkedList[T](Sized): ... -When inheriting from generic classes, some type variables could be fixed:: +When inheriting from generic classes, some type parameters could be fixed:: from collections.abc import Mapping - from typing import TypeVar - - T = TypeVar('T') - class MyDict(Mapping[str, T]): + class MyDict[T](Mapping[str, T]): ... In this case ``MyDict`` has a single parameter, ``T``. Using a generic class without specifying type parameters assumes :data:`Any` for each position. In the following example, ``MyIterable`` is -not generic but implicitly inherits from ``Iterable[Any]``:: +not generic but implicitly inherits from ``Iterable[Any]``: + +.. testcode:: from collections.abc import Iterable class MyIterable(Iterable): # Same as Iterable[Any] + ... -User defined generic type aliases are also supported. Examples:: +User-defined generic type aliases are also supported. Examples:: from collections.abc import Iterable - from typing import TypeVar - S = TypeVar('S') - Response = Iterable[S] | int + + type Response[S] = Iterable[S] | int # Return type here is same as Iterable[str] | int def response(query: str) -> Response[str]: ... - T = TypeVar('T', int, float, complex) - Vec = Iterable[tuple[T, T]] + type Vec[T] = Iterable[tuple[T, T]] - def inproduct(v: Vec[T]) -> T: # Same as Iterable[tuple[T, T]] + def inproduct[T: (int, float, complex)](v: Vec[T]) -> T: # Same as Iterable[tuple[T, T]] return sum(x*y for x, y in v) +For backward compatibility, generic type aliases can also be created +through a simple assignment:: + + from collections.abc import Iterable + from typing import TypeVar + + S = TypeVar("S") + Response = Iterable[S] | int + .. versionchanged:: 3.7 :class:`Generic` no longer has a custom metaclass. +.. versionchanged:: 3.12 + Syntactic support for generics and type aliases is new in version 3.12. + Previously, generic classes had to explicitly inherit from :class:`Generic` + or contain a type variable in one of their bases. + User-defined generics for parameter expressions are also supported via parameter -specification variables in the form ``Generic[P]``. The behavior is consistent +specification variables in the form ``[**P]``. The behavior is consistent with type variables' described above as parameter specification variables are treated by the typing module as a specialized type variable. The one exception to this is that a list of types can be used to substitute a :class:`ParamSpec`:: - >>> from typing import Generic, ParamSpec, TypeVar - - >>> T = TypeVar('T') - >>> P = ParamSpec('P') - - >>> class Z(Generic[T, P]): ... + >>> class Z[T, **P]: ... # T is a TypeVar; P is a ParamSpec ... >>> Z[int, [dict, float]] __main__.Z[int, [dict, float]] +Classes generic over a :class:`ParamSpec` can also be created using explicit +inheritance from :class:`Generic`. In this case, ``**`` is not used:: -Furthermore, a generic with only one parameter specification variable will accept + from typing import ParamSpec, Generic + + P = ParamSpec('P') + + class Z(Generic[P]): + ... + +Another difference between :class:`TypeVar` and :class:`ParamSpec` is that a +generic with only one parameter specification variable will accept parameter lists in the forms ``X[[Type1, Type2, ...]]`` and also ``X[Type1, Type2, ...]`` for aesthetic reasons. Internally, the latter is converted to the former, so the following are equivalent:: - >>> class X(Generic[P]): ... + >>> class X[**P]: ... ... >>> X[int, str] __main__.X[[int, str]] >>> X[[int, str]] __main__.X[[int, str]] -Do note that generics with :class:`ParamSpec` may not have correct +Note that generics with :class:`ParamSpec` may not have correct ``__parameters__`` after substitution in some cases because they are intended primarily for static type checking. @@ -576,25 +806,7 @@ can define new custom protocols to fully enjoy structural subtyping Module contents =============== -The module defines the following classes, functions and decorators. - -.. note:: - - This module defines several types that are subclasses of pre-existing - standard library classes which also extend :class:`Generic` - to support type variables inside ``[]``. - These types became redundant in Python 3.9 when the - corresponding pre-existing classes were enhanced to support ``[]``. - - The redundant types are deprecated as of Python 3.9 but no - deprecation warnings will be issued by the interpreter. - It is expected that type checkers will flag the deprecated types - when the checked program targets Python 3.9 or newer. - - The deprecated types will be removed from the :mod:`typing` module - no sooner than the first Python version released 5 years after the release of Python 3.9.0. - See details in :pep:`585`—*Type Hinting Generics In Standard Collections*. - +The ``typing`` module defines the following classes, functions and decorators. Special typing primitives ------------------------- @@ -602,7 +814,8 @@ Special typing primitives Special types """"""""""""" -These can be used as types in annotations and do not support ``[]``. +These can be used as types in annotations. They do not support subscription +using ``[]``. .. data:: Any @@ -616,29 +829,78 @@ These can be used as types in annotations and do not support ``[]``. avoiding type checker errors with classes that can duck type anywhere or are highly dynamic. +.. data:: AnyStr + + A :ref:`constrained type variable `. + + Definition:: + + AnyStr = TypeVar('AnyStr', str, bytes) + + ``AnyStr`` is meant to be used for functions that may accept :class:`str` or + :class:`bytes` arguments but cannot allow the two to mix. + + For example:: + + def concat(a: AnyStr, b: AnyStr) -> AnyStr: + return a + b + + concat("foo", "bar") # OK, output has type 'str' + concat(b"foo", b"bar") # OK, output has type 'bytes' + concat("foo", b"bar") # Error, cannot mix str and bytes + + Note that, despite its name, ``AnyStr`` has nothing to do with the + :class:`Any` type, nor does it mean "any string". In particular, ``AnyStr`` + and ``str | bytes`` are different from each other and have different use + cases:: + + # Invalid use of AnyStr: + # The type variable is used only once in the function signature, + # so cannot be "solved" by the type checker + def greet_bad(cond: bool) -> AnyStr: + return "hi there!" if cond else b"greetings!" + + # The better way of annotating this function: + def greet_proper(cond: bool) -> str | bytes: + return "hi there!" if cond else b"greetings!" + + .. deprecated-removed:: 3.13 3.18 + Deprecated in favor of the new :ref:`type parameter syntax `. + Use ``class A[T: (str, bytes)]: ...`` instead of importing ``AnyStr``. See + :pep:`695` for more details. + + In Python 3.16, ``AnyStr`` will be removed from ``typing.__all__``, and + deprecation warnings will be emitted at runtime when it is accessed or + imported from ``typing``. ``AnyStr`` will be removed from ``typing`` + in Python 3.18. + .. data:: LiteralString - Special type that includes only literal strings. A string + Special type that includes only literal strings. + + Any string literal is compatible with ``LiteralString``, as is another - ``LiteralString``, but an object typed as just ``str`` is not. + ``LiteralString``. However, an object typed as just ``str`` is not. A string created by composing ``LiteralString``-typed objects is also acceptable as a ``LiteralString``. - Example:: + Example: + + .. testcode:: - def run_query(sql: LiteralString) -> ... + def run_query(sql: LiteralString) -> None: ... def caller(arbitrary_string: str, literal_string: LiteralString) -> None: - run_query("SELECT * FROM students") # ok - run_query(literal_string) # ok - run_query("SELECT * FROM " + literal_string) # ok + run_query("SELECT * FROM students") # OK + run_query(literal_string) # OK + run_query("SELECT * FROM " + literal_string) # OK run_query(arbitrary_string) # type checker error run_query( # type checker error f"SELECT * FROM students WHERE name = {arbitrary_string}" ) - This is useful for sensitive APIs where arbitrary user-generated + ``LiteralString`` is useful for sensitive APIs where arbitrary user-generated strings could generate problems. For example, the two cases above that generate type checker errors could be vulnerable to an SQL injection attack. @@ -655,20 +917,20 @@ These can be used as types in annotations and do not support ``[]``. This can be used to define a function that should never be called, or a function that never returns:: - from typing import Never + from typing import Never - def never_call_me(arg: Never) -> None: - pass + def never_call_me(arg: Never) -> None: + pass - def int_or_str(arg: int | str) -> None: - never_call_me(arg) # type checker error - match arg: - case int(): - print("It's an int") - case str(): - print("It's a str") - case _: - never_call_me(arg) # ok, arg is of type Never + def int_or_str(arg: int | str) -> None: + never_call_me(arg) # type checker error + match arg: + case int(): + print("It's an int") + case str(): + print("It's a str") + case _: + never_call_me(arg) # OK, arg is of type Never .. versionadded:: 3.11 @@ -678,6 +940,7 @@ These can be used as types in annotations and do not support ``[]``. .. data:: NoReturn Special type indicating that a function never returns. + For example:: from typing import NoReturn @@ -697,15 +960,20 @@ These can be used as types in annotations and do not support ``[]``. .. data:: Self Special type to represent the current enclosed class. + For example:: - from typing import Self + from typing import Self, reveal_type class Foo: - def return_self(self) -> Self: - ... - return self + def return_self(self) -> Self: + ... + return self + + class SubclassOfFoo(Foo): pass + reveal_type(Foo().return_self()) # Revealed type is "Foo" + reveal_type(SubclassOfFoo().return_self()) # Revealed type is "SubclassOfFoo" This annotation is semantically equivalent to the following, albeit in a more succinct fashion:: @@ -715,19 +983,15 @@ These can be used as types in annotations and do not support ``[]``. Self = TypeVar("Self", bound="Foo") class Foo: - def return_self(self: Self) -> Self: - ... - return self - - In general if something currently follows the pattern of:: - - class Foo: - def return_self(self) -> "Foo": - ... - return self + def return_self(self: Self) -> Self: + ... + return self - You should use :data:`Self` as calls to ``SubclassOfFoo.return_self`` would have - ``Foo`` as the return type and not ``SubclassOfFoo``. + In general, if something returns ``self``, as in the above examples, you + should use ``Self`` as the return annotation. If ``Foo.return_self`` was + annotated as returning ``"Foo"``, then the type checker would infer the + object returned from ``SubclassOfFoo.return_self`` as being of type ``Foo`` + rather than ``SubclassOfFoo``. Other common use cases include: @@ -735,6 +999,17 @@ These can be used as types in annotations and do not support ``[]``. of the ``cls`` parameter. - Annotating an :meth:`~object.__enter__` method which returns self. + You should not use ``Self`` as the return annotation if the method is not + guaranteed to return an instance of a subclass when the class is + subclassed:: + + class Eggs: + # Self would be an incorrect return annotation here, + # as the object returned is always an instance of Eggs, + # even in subclasses + def returns_eggs(self) -> "Eggs": + return Eggs() + See :pep:`673` for more details. .. versionadded:: 3.11 @@ -742,38 +1017,52 @@ These can be used as types in annotations and do not support ``[]``. .. data:: TypeAlias Special annotation for explicitly declaring a :ref:`type alias `. + For example:: - from typing import TypeAlias + from typing import TypeAlias - Factors: TypeAlias = list[int] + Factors: TypeAlias = list[int] - See :pep:`613` for more details about explicit type aliases. + ``TypeAlias`` is particularly useful on older Python versions for annotating + aliases that make use of forward references, as it can be hard for type + checkers to distinguish these from normal variable assignments: - .. versionadded:: 3.10 + .. testcode:: -Special forms -""""""""""""" + from typing import Generic, TypeAlias, TypeVar -These can be used as types in annotations using ``[]``, each having a unique syntax. + T = TypeVar("T") -.. data:: Tuple + # "Box" does not exist yet, + # so we have to use quotes for the forward reference on Python <3.12. + # Using ``TypeAlias`` tells the type checker that this is a type alias declaration, + # not a variable assignment to a string. + BoxOfStrings: TypeAlias = "Box[str]" - Tuple type; ``Tuple[X, Y]`` is the type of a tuple of two items - with the first item of type X and the second of type Y. The type of - the empty tuple can be written as ``Tuple[()]``. + class Box(Generic[T]): + @classmethod + def make_box_of_strings(cls) -> BoxOfStrings: ... - Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding - to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple - of an int, a float and a string. + See :pep:`613` for more details. - To specify a variable-length tuple of homogeneous type, - use literal ellipsis, e.g. ``Tuple[int, ...]``. A plain :data:`Tuple` - is equivalent to ``Tuple[Any, ...]``, and in turn to :class:`tuple`. + .. versionadded:: 3.10 - .. deprecated:: 3.9 - :class:`builtins.tuple ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. deprecated:: 3.12 + :data:`TypeAlias` is deprecated in favor of the :keyword:`type` statement, + which creates instances of :class:`TypeAliasType` + and which natively supports forward references. + Note that while :data:`TypeAlias` and :class:`TypeAliasType` serve + similar purposes and have similar names, they are distinct and the + latter is not the type of the former. + Removal of :data:`TypeAlias` is not currently planned, but users + are encouraged to migrate to :keyword:`type` statements. + +Special forms +""""""""""""" + +These can be used as types in annotations. They all support subscription using +``[]``, but each has a unique syntax. .. data:: Union @@ -812,8 +1101,6 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. data:: Optional - Optional type. - ``Optional[X]`` is equivalent to ``X | None`` (or ``Union[X, None]``). Note that this is not the same concept as an optional argument, @@ -835,50 +1122,16 @@ These can be used as types in annotations using ``[]``, each having a unique syn Optional can now be written as ``X | None``. See :ref:`union type expressions`. -.. data:: Callable - - Callable type; ``Callable[[int], str]`` is a function of (int) -> str. - - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or an ellipsis; the return type must be - a single type. - - There is no syntax to indicate optional or keyword arguments; - such function types are rarely used as callback types. - ``Callable[..., ReturnType]`` (literal ellipsis) can be used to - type hint a callable taking any number of arguments and returning - ``ReturnType``. A plain :data:`Callable` is equivalent to - ``Callable[..., Any]``, and in turn to - :class:`collections.abc.Callable`. - - Callables which take other callables as arguments may indicate that their - parameter types are dependent on each other using :class:`ParamSpec`. - Additionally, if that callable adds or removes arguments from other - callables, the :data:`Concatenate` operator may be used. They - take the form ``Callable[ParamSpecVariable, ReturnType]`` and - ``Callable[Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable], ReturnType]`` - respectively. - - .. deprecated:: 3.9 - :class:`collections.abc.Callable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - - .. versionchanged:: 3.10 - ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. - See :pep:`612` for more details. - - .. seealso:: - The documentation for :class:`ParamSpec` and :class:`Concatenate` provide - examples of usage with ``Callable``. - .. data:: Concatenate - Used with :data:`Callable` and :class:`ParamSpec` to type annotate a higher - order callable which adds, removes, or transforms parameters of another + Special form for annotating higher-order functions. + + ``Concatenate`` can be used in conjunction with :ref:`Callable ` and + :class:`ParamSpec` to annotate a higher-order callable which adds, removes, + or transforms parameters of another callable. Usage is in the form ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` - is currently only valid when used as the first argument to a :data:`Callable`. + is currently only valid when used as the first argument to a :ref:`Callable `. The last parameter to ``Concatenate`` must be a :class:`ParamSpec` or ellipsis (``...``). @@ -892,16 +1145,13 @@ These can be used as types in annotations using ``[]``, each having a unique syn from collections.abc import Callable from threading import Lock - from typing import Concatenate, ParamSpec, TypeVar - - P = ParamSpec('P') - R = TypeVar('R') + from typing import Concatenate # Use this lock to ensure that only one thread is executing a function # at any time. my_lock = Lock() - def with_lock(f: Callable[Concatenate[Lock, P], R]) -> Callable[P, R]: + def with_lock[**P, R](f: Callable[Concatenate[Lock, P], R]) -> Callable[P, R]: '''A type-safe decorator which provides a lock.''' def inner(*args: P.args, **kwargs: P.kwargs) -> R: # Provide the lock as the first argument. @@ -917,75 +1167,33 @@ These can be used as types in annotations using ``[]``, each having a unique syn # We don't need to pass in the lock ourselves thanks to the decorator. sum_threadsafe([1.1, 2.2, 3.3]) -.. versionadded:: 3.10 - -.. seealso:: - - * :pep:`612` -- Parameter Specification Variables (the PEP which introduced - ``ParamSpec`` and ``Concatenate``). - * :class:`ParamSpec` and :class:`Callable`. - - -.. class:: Type(Generic[CT_co]) - - A variable annotated with ``C`` may accept a value of type ``C``. In - contrast, a variable annotated with ``Type[C]`` may accept values that are - classes themselves -- specifically, it will accept the *class object* of - ``C``. For example:: - - a = 3 # Has type 'int' - b = int # Has type 'Type[int]' - c = type(a) # Also has type 'Type[int]' - - Note that ``Type[C]`` is covariant:: - - class User: ... - class BasicUser(User): ... - class ProUser(User): ... - class TeamUser(User): ... - - # Accepts User, BasicUser, ProUser, TeamUser, ... - def make_new_user(user_class: Type[User]) -> User: - # ... - return user_class() - - The fact that ``Type[C]`` is covariant implies that all subclasses of - ``C`` should implement the same constructor signature and class method - signatures as ``C``. The type checker should flag violations of this, - but should also allow constructor calls in subclasses that match the - constructor calls in the indicated base class. How the type checker is - required to handle this particular case may change in future revisions of - :pep:`484`. - - The only legal parameters for :class:`Type` are classes, :data:`Any`, - :ref:`type variables `, and unions of any of these types. - For example:: + .. versionadded:: 3.10 - def new_non_team_user(user_class: Type[BasicUser | ProUser]): ... + .. seealso:: - ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent - to ``type``, which is the root of Python's metaclass hierarchy. + * :pep:`612` -- Parameter Specification Variables (the PEP which introduced + ``ParamSpec`` and ``Concatenate``) + * :class:`ParamSpec` + * :ref:`annotating-callables` - .. versionadded:: 3.5.2 +.. data:: Literal - .. deprecated:: 3.9 - :class:`builtins.type ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Special typing form to define "literal types". -.. data:: Literal + ``Literal`` can be used to indicate to type checkers that the + annotated object has a value equivalent to one of the + provided literals. - A type that can be used to indicate to type checkers that the - corresponding variable or function parameter has a value equivalent to - the provided literal (or one of several literals). For example:: + For example:: def validate_simple(data: Any) -> Literal[True]: # always returns True ... - MODE = Literal['r', 'rb', 'w', 'wb'] - def open_helper(file: str, mode: MODE) -> str: + type Mode = Literal['r', 'rb', 'w', 'wb'] + def open_helper(file: str, mode: Mode) -> str: ... - open_helper('/some/path', 'r') # Passes type check + open_helper('/some/path', 'r') # Passes type check open_helper('/other/path', 'typo') # Error in type checker ``Literal[...]`` cannot be subclassed. At runtime, an arbitrary value @@ -1028,8 +1236,12 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. data:: Final - A special typing construct to indicate to type checkers that a name - cannot be re-assigned or overridden in a subclass. For example:: + Special typing construct to indicate final names to type checkers. + + Final names cannot be reassigned in any scope. Final names declared in class + scopes cannot be overridden in subclasses. + + For example:: MAX_SIZE: Final = 9000 MAX_SIZE += 1 # Error reported by type checker @@ -1047,10 +1259,17 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. data:: Required + Special typing construct to mark a :class:`TypedDict` key as required. + + This is mainly useful for ``total=False`` TypedDicts. See :class:`TypedDict` + and :pep:`655` for more details. + + .. versionadded:: 3.11 + .. data:: NotRequired - Special typing constructs that mark individual keys of a :class:`TypedDict` - as either required or non-required respectively. + Special typing construct to mark a :class:`TypedDict` key as potentially + missing. See :class:`TypedDict` and :pep:`655` for more details. @@ -1058,92 +1277,150 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. data:: Annotated - A type, introduced in :pep:`593` (``Flexible function and variable - annotations``), to decorate existing types with context-specific metadata - (possibly multiple pieces of it, as ``Annotated`` is variadic). - Specifically, a type ``T`` can be annotated with metadata ``x`` via the - typehint ``Annotated[T, x]``. This metadata can be used for either static - analysis or at runtime. If a library (or tool) encounters a typehint - ``Annotated[T, x]`` and has no special logic for metadata ``x``, it - should ignore it and simply treat the type as ``T``. Unlike the - ``no_type_check`` functionality that currently exists in the ``typing`` - module which completely disables typechecking annotations on a function - or a class, the ``Annotated`` type allows for both static typechecking - of ``T`` (which can safely ignore ``x``) - together with runtime access to ``x`` within a specific application. - - Ultimately, the responsibility of how to interpret the annotations (if - at all) is the responsibility of the tool or library encountering the - ``Annotated`` type. A tool or library encountering an ``Annotated`` type - can scan through the annotations to determine if they are of interest - (e.g., using ``isinstance()``). - - When a tool or a library does not support annotations or encounters an - unknown annotation it should just ignore it and treat annotated type as - the underlying type. - - It's up to the tool consuming the annotations to decide whether the - client is allowed to have several annotations on one type and how to - merge those annotations. - - Since the ``Annotated`` type allows you to put several annotations of - the same (or different) type(s) on any node, the tools or libraries - consuming those annotations are in charge of dealing with potential - duplicates. For example, if you are doing value range analysis you might - allow this:: - - T1 = Annotated[int, ValueRange(-10, 5)] - T2 = Annotated[T1, ValueRange(-20, 3)] - - Passing ``include_extras=True`` to :func:`get_type_hints` lets one - access the extra annotations at runtime. - - The details of the syntax: + Special typing form to add context-specific metadata to an annotation. + + Add metadata ``x`` to a given type ``T`` by using the annotation + ``Annotated[T, x]``. Metadata added using ``Annotated`` can be used by + static analysis tools or at runtime. At runtime, the metadata is stored + in a :attr:`!__metadata__` attribute. + + If a library or tool encounters an annotation ``Annotated[T, x]`` and has + no special logic for the metadata, it should ignore the metadata and simply + treat the annotation as ``T``. As such, ``Annotated`` can be useful for code + that wants to use annotations for purposes outside Python's static typing + system. + + Using ``Annotated[T, x]`` as an annotation still allows for static + typechecking of ``T``, as type checkers will simply ignore the metadata ``x``. + In this way, ``Annotated`` differs from the + :func:`@no_type_check ` decorator, which can also be used for + adding annotations outside the scope of the typing system, but + completely disables typechecking for a function or class. + + The responsibility of how to interpret the metadata + lies with the tool or library encountering an + ``Annotated`` annotation. A tool or library encountering an ``Annotated`` type + can scan through the metadata elements to determine if they are of interest + (e.g., using :func:`isinstance`). + + .. describe:: Annotated[, ] + + Here is an example of how you might use ``Annotated`` to add metadata to + type annotations if you were doing range analysis: + + .. testcode:: + + @dataclass + class ValueRange: + lo: int + hi: int + + T1 = Annotated[int, ValueRange(-10, 5)] + T2 = Annotated[T1, ValueRange(-20, 3)] + + Details of the syntax: * The first argument to ``Annotated`` must be a valid type - * Multiple type annotations are supported (``Annotated`` supports variadic + * Multiple metadata elements can be supplied (``Annotated`` supports variadic arguments):: - Annotated[int, ValueRange(3, 10), ctype("char")] + @dataclass + class ctype: + kind: str + + Annotated[int, ValueRange(3, 10), ctype("char")] - * ``Annotated`` must be called with at least two arguments ( + It is up to the tool consuming the annotations to decide whether the + client is allowed to add multiple metadata elements to one annotation and how to + merge those annotations. + + * ``Annotated`` must be subscripted with at least two arguments ( ``Annotated[int]`` is not valid) - * The order of the annotations is preserved and matters for equality + * The order of the metadata elements is preserved and matters for equality checks:: - Annotated[int, ValueRange(3, 10), ctype("char")] != Annotated[ - int, ctype("char"), ValueRange(3, 10) - ] + assert Annotated[int, ValueRange(3, 10), ctype("char")] != Annotated[ + int, ctype("char"), ValueRange(3, 10) + ] + + * Nested ``Annotated`` types are flattened. The order of the metadata elements + starts with the innermost annotation:: + + assert Annotated[Annotated[int, ValueRange(3, 10)], ctype("char")] == Annotated[ + int, ValueRange(3, 10), ctype("char") + ] + + * Duplicated metadata elements are not removed:: + + assert Annotated[int, ValueRange(3, 10)] != Annotated[ + int, ValueRange(3, 10), ValueRange(3, 10) + ] + + * ``Annotated`` can be used with nested and generic aliases: + + .. testcode:: + + @dataclass + class MaxLen: + value: int + + type Vec[T] = Annotated[list[tuple[T, T]], MaxLen(10)] + + # When used in a type annotation, a type checker will treat "V" the same as + # ``Annotated[list[tuple[int, int]], MaxLen(10)]``: + type V = Vec[int] + + * ``Annotated`` cannot be used with an unpacked :class:`TypeVarTuple`:: + + type Variadic[*Ts] = Annotated[*Ts, Ann1] # NOT valid - * Nested ``Annotated`` types are flattened, with metadata ordered - starting with the innermost annotation:: + This would be equivalent to:: - Annotated[Annotated[int, ValueRange(3, 10)], ctype("char")] == Annotated[ - int, ValueRange(3, 10), ctype("char") - ] + Annotated[T1, T2, T3, ..., Ann1] - * Duplicated annotations are not removed:: + where ``T1``, ``T2``, etc. are :class:`TypeVars `. This would be + invalid: only one type should be passed to Annotated. - Annotated[int, ValueRange(3, 10)] != Annotated[ - int, ValueRange(3, 10), ValueRange(3, 10) - ] + * By default, :func:`get_type_hints` strips the metadata from annotations. + Pass ``include_extras=True`` to have the metadata preserved: - * ``Annotated`` can be used with nested and generic aliases:: + .. doctest:: - T = TypeVar('T') - Vec = Annotated[list[tuple[T, T]], MaxLen(10)] - V = Vec[int] + >>> from typing import Annotated, get_type_hints + >>> def func(x: Annotated[int, "metadata"]) -> None: pass + ... + >>> get_type_hints(func) + {'x': , 'return': } + >>> get_type_hints(func, include_extras=True) + {'x': typing.Annotated[int, 'metadata'], 'return': } + + * At runtime, the metadata associated with an ``Annotated`` type can be + retrieved via the :attr:`!__metadata__` attribute: + + .. doctest:: + + >>> from typing import Annotated + >>> X = Annotated[int, "very", "important", "metadata"] + >>> X + typing.Annotated[int, 'very', 'important', 'metadata'] + >>> X.__metadata__ + ('very', 'important', 'metadata') - V == Annotated[list[tuple[int, int]], MaxLen(10)] + .. seealso:: + + :pep:`593` - Flexible function and variable annotations + The PEP introducing ``Annotated`` to the standard library. .. versionadded:: 3.9 .. data:: TypeGuard - Special typing form used to annotate the return type of a user-defined + Special typing construct for marking user-defined type guard functions. + + ``TypeGuard`` can be used to annotate the return type of a user-defined type guard function. ``TypeGuard`` only accepts a single type argument. At runtime, functions marked this way should return a boolean. @@ -1208,256 +1485,379 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionadded:: 3.10 -Building generic types -"""""""""""""""""""""" - -These are not used in annotations. They are building blocks for creating generic types. +.. data:: Unpack -.. class:: Generic + Typing operator to conceptually mark an object as having been unpacked. - Abstract base class for generic types. + For example, using the unpack operator ``*`` on a + :ref:`type variable tuple ` is equivalent to using ``Unpack`` + to mark the type variable tuple as having been unpacked:: - A generic type is typically declared by inheriting from an - instantiation of this class with one or more type variables. - For example, a generic mapping type might be defined as:: + Ts = TypeVarTuple('Ts') + tup: tuple[*Ts] + # Effectively does: + tup: tuple[Unpack[Ts]] - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. + In fact, ``Unpack`` can be used interchangeably with ``*`` in the context + of :class:`typing.TypeVarTuple ` and + :class:`builtins.tuple ` types. You might see ``Unpack`` being used + explicitly in older versions of Python, where ``*`` couldn't be used in + certain places:: - This class can then be used as follows:: + # In older versions of Python, TypeVarTuple and Unpack + # are located in the `typing_extensions` backports package. + from typing_extensions import TypeVarTuple, Unpack - X = TypeVar('X') - Y = TypeVar('Y') + Ts = TypeVarTuple('Ts') + tup: tuple[*Ts] # Syntax error on Python <= 3.10! + tup: tuple[Unpack[Ts]] # Semantically equivalent, and backwards-compatible - def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y: + ``Unpack`` can also be used along with :class:`typing.TypedDict` for typing + ``**kwargs`` in a function signature:: + + from typing import TypedDict, Unpack + + class Movie(TypedDict): + name: str + year: int + + # This function expects two keyword arguments - `name` of type `str` + # and `year` of type `int`. + def foo(**kwargs: Unpack[Movie]): ... + + See :pep:`692` for more details on using ``Unpack`` for ``**kwargs`` typing. + + .. versionadded:: 3.11 + +Building generic types and type aliases +""""""""""""""""""""""""""""""""""""""" + +The following classes should not be used directly as annotations. +Their intended purpose is to be building blocks +for creating generic types and type aliases. + +These objects can be created through special syntax +(:ref:`type parameter lists ` and the :keyword:`type` statement). +For compatibility with Python 3.11 and earlier, they can also be created +without the dedicated syntax, as documented below. + +.. class:: Generic + + Abstract base class for generic types. + + A generic type is typically declared by adding a list of type parameters + after the class name:: + + class Mapping[KT, VT]: + def __getitem__(self, key: KT) -> VT: + ... + # Etc. + + Such a class implicitly inherits from ``Generic``. + The runtime semantics of this syntax are discussed in the + :ref:`Language Reference `. + + This class can then be used as follows:: + + def lookup_name[X, Y](mapping: Mapping[X, Y], key: X, default: Y) -> Y: try: return mapping[key] except KeyError: return default -.. class:: TypeVar + Here the brackets after the function name indicate a + :ref:`generic function `. - Type variable. + For backwards compatibility, generic classes can also be + declared by explicitly inheriting from + ``Generic``. In this case, the type parameters must be declared + separately:: - Usage:: + KT = TypeVar('KT') + VT = TypeVar('VT') + + class Mapping(Generic[KT, VT]): + def __getitem__(self, key: KT) -> VT: + ... + # Etc. + +.. _typevar: + +.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False) + + Type variable. + + The preferred way to construct a type variable is via the dedicated syntax + for :ref:`generic functions `, + :ref:`generic classes `, and + :ref:`generic type aliases `:: + + class Sequence[T]: # T is a TypeVar + ... + + This syntax can also be used to create bound and constrained type + variables:: + + class StrSequence[S: str]: # S is a TypeVar bound to str + ... + + + class StrOrBytesSequence[A: (str, bytes)]: # A is a TypeVar constrained to str or bytes + ... + + However, if desired, reusable type variables can also be constructed manually, like so:: T = TypeVar('T') # Can be anything S = TypeVar('S', bound=str) # Can be any subtype of str A = TypeVar('A', str, bytes) # Must be exactly str or bytes - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See :class:`Generic` for more - information on generic types. Generic functions work as follows:: + Type variables exist primarily for the benefit of static type + checkers. They serve as the parameters for generic types as well + as for generic function and type alias definitions. + See :class:`Generic` for more + information on generic types. Generic functions work as follows:: - def repeat(x: T, n: int) -> Sequence[T]: - """Return a list containing n references to x.""" - return [x]*n + def repeat[T](x: T, n: int) -> Sequence[T]: + """Return a list containing n references to x.""" + return [x]*n - def print_capitalized(x: S) -> S: - """Print x capitalized, and return x.""" - print(x.capitalize()) - return x + def print_capitalized[S: str](x: S) -> S: + """Print x capitalized, and return x.""" + print(x.capitalize()) + return x - def concatenate(x: A, y: A) -> A: - """Add two strings or bytes objects together.""" - return x + y + def concatenate[A: (str, bytes)](x: A, y: A) -> A: + """Add two strings or bytes objects together.""" + return x + y - Note that type variables can be *bound*, *constrained*, or neither, but - cannot be both bound *and* constrained. + Note that type variables can be *bound*, *constrained*, or neither, but + cannot be both bound *and* constrained. - Bound type variables and constrained type variables have different - semantics in several important ways. Using a *bound* type variable means - that the ``TypeVar`` will be solved using the most specific type possible:: + The variance of type variables is inferred by type checkers when they are created + through the :ref:`type parameter syntax ` or when + ``infer_variance=True`` is passed. + Manually created type variables may be explicitly marked covariant or contravariant by passing + ``covariant=True`` or ``contravariant=True``. + By default, manually created type variables are invariant. + See :pep:`484` and :pep:`695` for more details. - x = print_capitalized('a string') - reveal_type(x) # revealed type is str + Bound type variables and constrained type variables have different + semantics in several important ways. Using a *bound* type variable means + that the ``TypeVar`` will be solved using the most specific type possible:: - class StringSubclass(str): - pass + x = print_capitalized('a string') + reveal_type(x) # revealed type is str - y = print_capitalized(StringSubclass('another string')) - reveal_type(y) # revealed type is StringSubclass + class StringSubclass(str): + pass - z = print_capitalized(45) # error: int is not a subtype of str + y = print_capitalized(StringSubclass('another string')) + reveal_type(y) # revealed type is StringSubclass - Type variables can be bound to concrete types, abstract types (ABCs or - protocols), and even unions of types:: + z = print_capitalized(45) # error: int is not a subtype of str - U = TypeVar('U', bound=str|bytes) # Can be any subtype of the union str|bytes - V = TypeVar('V', bound=SupportsAbs) # Can be anything with an __abs__ method + Type variables can be bound to concrete types, abstract types (ABCs or + protocols), and even unions of types:: -.. _typing-constrained-typevar: + # Can be anything with an __abs__ method + def print_abs[T: SupportsAbs](arg: T) -> None: + print("Absolute value:", abs(arg)) - Using a *constrained* type variable, however, means that the ``TypeVar`` - can only ever be solved as being exactly one of the constraints given:: + U = TypeVar('U', bound=str|bytes) # Can be any subtype of the union str|bytes + V = TypeVar('V', bound=SupportsAbs) # Can be anything with an __abs__ method - a = concatenate('one', 'two') - reveal_type(a) # revealed type is str + .. _typing-constrained-typevar: - b = concatenate(StringSubclass('one'), StringSubclass('two')) - reveal_type(b) # revealed type is str, despite StringSubclass being passed in + Using a *constrained* type variable, however, means that the ``TypeVar`` + can only ever be solved as being exactly one of the constraints given:: - c = concatenate('one', b'two') # error: type variable 'A' can be either str or bytes in a function call, but not both + a = concatenate('one', 'two') + reveal_type(a) # revealed type is str - At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. In general, - :func:`isinstance` and :func:`issubclass` should not be used with types. + b = concatenate(StringSubclass('one'), StringSubclass('two')) + reveal_type(b) # revealed type is str, despite StringSubclass being passed in - Type variables may be marked covariant or contravariant by passing - ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more - details. By default, type variables are invariant. + c = concatenate('one', b'two') # error: type variable 'A' can be either str or bytes in a function call, but not both -.. class:: TypeVarTuple + At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. - Type variable tuple. A specialized form of :class:`type variable ` - that enables *variadic* generics. + .. attribute:: __name__ - A normal type variable enables parameterization with a single type. A type - variable tuple, in contrast, allows parameterization with an - *arbitrary* number of types by acting like an *arbitrary* number of type - variables wrapped in a tuple. For example:: + The name of the type variable. - T = TypeVar('T') - Ts = TypeVarTuple('Ts') + .. attribute:: __covariant__ - def move_first_element_to_last(tup: tuple[T, *Ts]) -> tuple[*Ts, T]: - return (*tup[1:], tup[0]) + Whether the type var has been explicitly marked as covariant. - # T is bound to int, Ts is bound to () - # Return value is (1,), which has type tuple[int] - move_first_element_to_last(tup=(1,)) + .. attribute:: __contravariant__ - # T is bound to int, Ts is bound to (str,) - # Return value is ('spam', 1), which has type tuple[str, int] - move_first_element_to_last(tup=(1, 'spam')) + Whether the type var has been explicitly marked as contravariant. - # T is bound to int, Ts is bound to (str, float) - # Return value is ('spam', 3.0, 1), which has type tuple[str, float, int] - move_first_element_to_last(tup=(1, 'spam', 3.0)) + .. attribute:: __infer_variance__ - # This fails to type check (and fails at runtime) - # because tuple[()] is not compatible with tuple[T, *Ts] - # (at least one element is required) - move_first_element_to_last(tup=()) + Whether the type variable's variance should be inferred by type checkers. - Note the use of the unpacking operator ``*`` in ``tuple[T, *Ts]``. - Conceptually, you can think of ``Ts`` as a tuple of type variables - ``(T1, T2, ...)``. ``tuple[T, *Ts]`` would then become - ``tuple[T, *(T1, T2, ...)]``, which is equivalent to - ``tuple[T, T1, T2, ...]``. (Note that in older versions of Python, you might - see this written using :data:`Unpack ` instead, as - ``Unpack[Ts]``.) + .. versionadded:: 3.12 - Type variable tuples must *always* be unpacked. This helps distinguish type - variable tuples from normal type variables:: + .. attribute:: __bound__ - x: Ts # Not valid - x: tuple[Ts] # Not valid - x: tuple[*Ts] # The correct way to do it + The bound of the type variable, if any. - Type variable tuples can be used in the same contexts as normal type - variables. For example, in class definitions, arguments, and return types:: + .. versionchanged:: 3.12 - Shape = TypeVarTuple('Shape') - class Array(Generic[*Shape]): - def __getitem__(self, key: tuple[*Shape]) -> float: ... - def __abs__(self) -> "Array[*Shape]": ... - def get_shape(self) -> tuple[*Shape]: ... + For type variables created through :ref:`type parameter syntax `, + the bound is evaluated only when the attribute is accessed, not when + the type variable is created (see :ref:`lazy-evaluation`). - Type variable tuples can be happily combined with normal type variables:: + .. attribute:: __constraints__ - DType = TypeVar('DType') + A tuple containing the constraints of the type variable, if any. - class Array(Generic[DType, *Shape]): # This is fine - pass + .. versionchanged:: 3.12 - class Array2(Generic[*Shape, DType]): # This would also be fine - pass + For type variables created through :ref:`type parameter syntax `, + the constraints are evaluated only when the attribute is accessed, not when + the type variable is created (see :ref:`lazy-evaluation`). - float_array_1d: Array[float, Height] = Array() # Totally fine - int_array_2d: Array[int, Height, Width] = Array() # Yup, fine too + .. versionchanged:: 3.12 - However, note that at most one type variable tuple may appear in a single - list of type arguments or type parameters:: + Type variables can now be declared using the + :ref:`type parameter ` syntax introduced by :pep:`695`. + The ``infer_variance`` parameter was added. - x: tuple[*Ts, *Ts] # Not valid - class Array(Generic[*Shape, *Shape]): # Not valid - pass +.. _typevartuple: - Finally, an unpacked type variable tuple can be used as the type annotation - of ``*args``:: +.. class:: TypeVarTuple(name) - def call_soon( - callback: Callable[[*Ts], None], - *args: *Ts - ) -> None: - ... - callback(*args) + Type variable tuple. A specialized form of :ref:`type variable ` + that enables *variadic* generics. - In contrast to non-unpacked annotations of ``*args`` - e.g. ``*args: int``, - which would specify that *all* arguments are ``int`` - ``*args: *Ts`` - enables reference to the types of the *individual* arguments in ``*args``. - Here, this allows us to ensure the types of the ``*args`` passed - to ``call_soon`` match the types of the (positional) arguments of - ``callback``. + Type variable tuples can be declared in :ref:`type parameter lists ` + using a single asterisk (``*``) before the name:: - See :pep:`646` for more details on type variable tuples. + def move_first_element_to_last[T, *Ts](tup: tuple[T, *Ts]) -> tuple[*Ts, T]: + return (*tup[1:], tup[0]) - .. versionadded:: 3.11 + Or by explicitly invoking the ``TypeVarTuple`` constructor:: -.. data:: Unpack + T = TypeVar("T") + Ts = TypeVarTuple("Ts") - A typing operator that conceptually marks an object as having been - unpacked. For example, using the unpack operator ``*`` on a - :class:`type variable tuple ` is equivalent to using ``Unpack`` - to mark the type variable tuple as having been unpacked:: + def move_first_element_to_last(tup: tuple[T, *Ts]) -> tuple[*Ts, T]: + return (*tup[1:], tup[0]) - Ts = TypeVarTuple('Ts') - tup: tuple[*Ts] - # Effectively does: - tup: tuple[Unpack[Ts]] + A normal type variable enables parameterization with a single type. A type + variable tuple, in contrast, allows parameterization with an + *arbitrary* number of types by acting like an *arbitrary* number of type + variables wrapped in a tuple. For example:: - In fact, ``Unpack`` can be used interchangeably with ``*`` in the context - of :class:`typing.TypeVarTuple ` and - :class:`builtins.tuple ` types. You might see ``Unpack`` being used - explicitly in older versions of Python, where ``*`` couldn't be used in - certain places:: + # T is bound to int, Ts is bound to () + # Return value is (1,), which has type tuple[int] + move_first_element_to_last(tup=(1,)) - # In older versions of Python, TypeVarTuple and Unpack - # are located in the `typing_extensions` backports package. - from typing_extensions import TypeVarTuple, Unpack + # T is bound to int, Ts is bound to (str,) + # Return value is ('spam', 1), which has type tuple[str, int] + move_first_element_to_last(tup=(1, 'spam')) - Ts = TypeVarTuple('Ts') - tup: tuple[*Ts] # Syntax error on Python <= 3.10! - tup: tuple[Unpack[Ts]] # Semantically equivalent, and backwards-compatible + # T is bound to int, Ts is bound to (str, float) + # Return value is ('spam', 3.0, 1), which has type tuple[str, float, int] + move_first_element_to_last(tup=(1, 'spam', 3.0)) - ``Unpack`` can also be used along with :class:`typing.TypedDict` for typing - ``**kwargs`` in a function signature:: + # This fails to type check (and fails at runtime) + # because tuple[()] is not compatible with tuple[T, *Ts] + # (at least one element is required) + move_first_element_to_last(tup=()) - from typing import TypedDict, Unpack + Note the use of the unpacking operator ``*`` in ``tuple[T, *Ts]``. + Conceptually, you can think of ``Ts`` as a tuple of type variables + ``(T1, T2, ...)``. ``tuple[T, *Ts]`` would then become + ``tuple[T, *(T1, T2, ...)]``, which is equivalent to + ``tuple[T, T1, T2, ...]``. (Note that in older versions of Python, you might + see this written using :data:`Unpack ` instead, as + ``Unpack[Ts]``.) - class Movie(TypedDict): - name: str - year: int + Type variable tuples must *always* be unpacked. This helps distinguish type + variable tuples from normal type variables:: - # This function expects two keyword arguments - `name` of type `str` - # and `year` of type `int`. - def foo(**kwargs: Unpack[Movie]): ... + x: Ts # Not valid + x: tuple[Ts] # Not valid + x: tuple[*Ts] # The correct way to do it - See :pep:`692` for more details on using ``Unpack`` for ``**kwargs`` typing. + Type variable tuples can be used in the same contexts as normal type + variables. For example, in class definitions, arguments, and return types:: + + class Array[*Shape]: + def __getitem__(self, key: tuple[*Shape]) -> float: ... + def __abs__(self) -> "Array[*Shape]": ... + def get_shape(self) -> tuple[*Shape]: ... + + Type variable tuples can be happily combined with normal type variables: + + .. testcode:: + + class Array[DType, *Shape]: # This is fine + pass + + class Array2[*Shape, DType]: # This would also be fine + pass + + class Height: ... + class Width: ... + + float_array_1d: Array[float, Height] = Array() # Totally fine + int_array_2d: Array[int, Height, Width] = Array() # Yup, fine too + + However, note that at most one type variable tuple may appear in a single + list of type arguments or type parameters:: + + x: tuple[*Ts, *Ts] # Not valid + class Array[*Shape, *Shape]: # Not valid + pass + + Finally, an unpacked type variable tuple can be used as the type annotation + of ``*args``:: + + def call_soon[*Ts]( + callback: Callable[[*Ts], None], + *args: *Ts + ) -> None: + ... + callback(*args) + + In contrast to non-unpacked annotations of ``*args`` - e.g. ``*args: int``, + which would specify that *all* arguments are ``int`` - ``*args: *Ts`` + enables reference to the types of the *individual* arguments in ``*args``. + Here, this allows us to ensure the types of the ``*args`` passed + to ``call_soon`` match the types of the (positional) arguments of + ``callback``. + + See :pep:`646` for more details on type variable tuples. + + .. attribute:: __name__ + + The name of the type variable tuple. .. versionadded:: 3.11 + .. versionchanged:: 3.12 + + Type variable tuples can now be declared using the + :ref:`type parameter ` syntax introduced by :pep:`695`. + .. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) Parameter specification variable. A specialized version of - :class:`type variables `. + :ref:`type variables `. - Usage:: + In :ref:`type parameter lists `, parameter specifications + can be declared with two asterisks (``**``):: + + type IntFunc[**P] = Callable[P, int] + + For compatibility with Python 3.11 and earlier, ``ParamSpec`` objects + can also be created as follows:: P = ParamSpec('P') @@ -1474,13 +1874,9 @@ These are not used in annotations. They are building blocks for creating generic new callable returned by it have inter-dependent type parameters:: from collections.abc import Callable - from typing import TypeVar, ParamSpec import logging - T = TypeVar('T') - P = ParamSpec('P') - - def add_logging(f: Callable[P, T]) -> Callable[P, T]: + def add_logging[T, **P](f: Callable[P, T]) -> Callable[P, T]: '''A type-safe decorator to add logging to a function.''' def inner(*args: P.args, **kwargs: P.kwargs) -> T: logging.info(f'{f.__name__} was called') @@ -1515,6 +1911,10 @@ These are not used in annotations. They are building blocks for creating generic ``P.args`` and ``P.kwargs`` are instances respectively of :class:`ParamSpecArgs` and :class:`ParamSpecKwargs`. + .. attribute:: __name__ + + The name of the parameter specification. + Parameter specification variables created with ``covariant=True`` or ``contravariant=True`` can be used to declare covariant or contravariant generic types. The ``bound`` argument is also accepted, similar to @@ -1523,14 +1923,20 @@ These are not used in annotations. They are building blocks for creating generic .. versionadded:: 3.10 + .. versionchanged:: 3.12 + + Parameter specifications can now be declared using the + :ref:`type parameter ` syntax introduced by :pep:`695`. + .. note:: Only parameter specification variables defined in global scope can be pickled. .. seealso:: * :pep:`612` -- Parameter Specification Variables (the PEP which introduced - ``ParamSpec`` and ``Concatenate``). - * :class:`Callable` and :class:`Concatenate`. + ``ParamSpec`` and ``Concatenate``) + * :data:`Concatenate` + * :ref:`annotating-callables` .. data:: ParamSpecArgs .. data:: ParamSpecKwargs @@ -1541,128 +1947,91 @@ These are not used in annotations. They are building blocks for creating generic for runtime introspection and have no special meaning to static type checkers. Calling :func:`get_origin` on either of these objects will return the - original ``ParamSpec``:: - - P = ParamSpec("P") - get_origin(P.args) # returns P - get_origin(P.kwargs) # returns P - - .. versionadded:: 3.10 - - -.. data:: AnyStr - - ``AnyStr`` is a :ref:`constrained type variable ` defined as - ``AnyStr = TypeVar('AnyStr', str, bytes)``. - - It is meant to be used for functions that may accept any kind of string - without allowing different kinds of strings to mix. For example:: - - def concat(a: AnyStr, b: AnyStr) -> AnyStr: - return a + b - - concat(u"foo", u"bar") # Ok, output has type 'unicode' - concat(b"foo", b"bar") # Ok, output has type 'bytes' - concat(u"foo", b"bar") # Error, cannot mix unicode and bytes - -.. class:: Protocol(Generic) + original ``ParamSpec``: - Base class for protocol classes. Protocol classes are defined like this:: + .. doctest:: - class Proto(Protocol): - def meth(self) -> int: - ... + >>> from typing import ParamSpec, get_origin + >>> P = ParamSpec("P") + >>> get_origin(P.args) is P + True + >>> get_origin(P.kwargs) is P + True - Such classes are primarily used with static type checkers that recognize - structural subtyping (static duck-typing), for example:: + .. versionadded:: 3.10 - class C: - def meth(self) -> int: - return 0 - def func(x: Proto) -> int: - return x.meth() +.. class:: TypeAliasType(name, value, *, type_params=()) - func(C()) # Passes static type check + The type of type aliases created through the :keyword:`type` statement. - See :pep:`544` for more details. Protocol classes decorated with - :func:`runtime_checkable` (described later) act as simple-minded runtime - protocols that check only the presence of given attributes, ignoring their - type signatures. + Example: - Protocol classes can be generic, for example:: + .. doctest:: - class GenProto(Protocol[T]): - def meth(self) -> T: - ... + >>> type Alias = int + >>> type(Alias) + - .. versionadded:: 3.8 + .. versionadded:: 3.12 -.. decorator:: runtime_checkable + .. attribute:: __name__ - Mark a protocol class as a runtime protocol. + The name of the type alias: - Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. - This raises :exc:`TypeError` when applied to a non-protocol class. This - allows a simple-minded structural check, very similar to "one trick ponies" - in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: + .. doctest:: - @runtime_checkable - class Closable(Protocol): - def close(self): ... + >>> type Alias = int + >>> Alias.__name__ + 'Alias' - assert isinstance(open('/some/file'), Closable) + .. attribute:: __module__ - @runtime_checkable - class Named(Protocol): - name: str + The module in which the type alias was defined:: - import threading - assert isinstance(threading.Thread(name='Bob'), Named) + >>> type Alias = int + >>> Alias.__module__ + '__main__' - .. note:: + .. attribute:: __type_params__ - :func:`!runtime_checkable` will check only the presence of the required - methods or attributes, not their type signatures or types. - For example, :class:`ssl.SSLObject` - is a class, therefore it passes an :func:`issubclass` - check against :data:`Callable`. However, the - ``ssl.SSLObject.__init__`` method exists only to raise a - :exc:`TypeError` with a more informative message, therefore making - it impossible to call (instantiate) :class:`ssl.SSLObject`. + The type parameters of the type alias, or an empty tuple if the alias is + not generic: - .. note:: + .. doctest:: - An :func:`isinstance` check against a runtime-checkable protocol can be - surprisingly slow compared to an ``isinstance()`` check against - a non-protocol class. Consider using alternative idioms such as - :func:`hasattr` calls for structural checks in performance-sensitive - code. + >>> type ListOrSet[T] = list[T] | set[T] + >>> ListOrSet.__type_params__ + (T,) + >>> type NotGeneric = int + >>> NotGeneric.__type_params__ + () - .. versionadded:: 3.8 + .. attribute:: __value__ - .. versionchanged:: 3.12 - The internal implementation of :func:`isinstance` checks against - runtime-checkable protocols now uses :func:`inspect.getattr_static` - to look up attributes (previously, :func:`hasattr` was used). - As a result, some objects which used to be considered instances - of a runtime-checkable protocol may no longer be considered instances - of that protocol on Python 3.12+, and vice versa. - Most users are unlikely to be affected by this change. + The type alias's value. This is :ref:`lazily evaluated `, + so names used in the definition of the alias are not resolved until the + ``__value__`` attribute is accessed: - .. versionchanged:: 3.12 - The members of a runtime-checkable protocol are now considered "frozen" - at runtime as soon as the class has been created. Monkey-patching - attributes onto a runtime-checkable protocol will still work, but will - have no impact on :func:`isinstance` checks comparing objects to the - protocol. See :ref:`"What's new in Python 3.12" ` - for more details. + .. doctest:: + >>> type Mutually = Recursive + >>> type Recursive = Mutually + >>> Mutually + Mutually + >>> Recursive + Recursive + >>> Mutually.__value__ + Recursive + >>> Recursive.__value__ + Mutually Other special directives """""""""""""""""""""""" -These are not used in annotations. They are building blocks for declaring types. +These functions and classes should not be used directly as annotations. +Their intended purpose is to be building blocks for creating and declaring +types. .. class:: NamedTuple @@ -1707,12 +2076,18 @@ These are not used in annotations. They are building blocks for declaring types. ``NamedTuple`` subclasses can be generic:: - class Group(NamedTuple, Generic[T]): + class Group[T](NamedTuple): key: T group: list[T] Backward-compatible usage:: + # For creating a generic NamedTuple on Python 3.11 or lower + class Group(NamedTuple, Generic[T]): + key: T + group: list[T] + + # A functional syntax is also supported Employee = NamedTuple('Employee', [('name', str), ('id', int)]) .. versionchanged:: 3.6 @@ -1732,80 +2107,201 @@ These are not used in annotations. They are building blocks for declaring types. .. versionchanged:: 3.11 Added support for generic namedtuples. + .. deprecated-removed:: 3.13 3.15 + The undocumented keyword argument syntax for creating NamedTuple classes + (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed + in 3.15. Use the class-based syntax or the functional syntax instead. + + .. deprecated-removed:: 3.13 3.15 + When using the functional syntax to create a NamedTuple class, failing to + pass a value to the 'fields' parameter (``NT = NamedTuple("NT")``) is + deprecated. Passing ``None`` to the 'fields' parameter + (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be + disallowed in Python 3.15. To create a NamedTuple class with 0 fields, + use ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. + .. class:: NewType(name, tp) - A helper class to indicate a distinct type to a typechecker, - see :ref:`distinct`. At runtime it returns an object that returns - its argument when called. + Helper class to create low-overhead :ref:`distinct types `. + + A ``NewType`` is considered a distinct type by a typechecker. At runtime, + however, calling a ``NewType`` returns its argument unchanged. + Usage:: - UserId = NewType('UserId', int) - first_user = UserId(1) + UserId = NewType('UserId', int) # Declare the NewType "UserId" + first_user = UserId(1) # "UserId" returns the argument unchanged at runtime + + .. attribute:: __module__ + + The module in which the new type is defined. + + .. attribute:: __name__ + + The name of the new type. + + .. attribute:: __supertype__ + + The type that the new type is based on. .. versionadded:: 3.5.2 .. versionchanged:: 3.10 ``NewType`` is now a class rather than a function. -.. class:: TypedDict(dict) +.. class:: Protocol(Generic) - Special construct to add type hints to a dictionary. - At runtime it is a plain :class:`dict`. + Base class for protocol classes. - ``TypedDict`` declares a dictionary type that expects all of its - instances to have a certain set of keys, where each key is - associated with a value of a consistent type. This expectation - is not checked at runtime but is only enforced by type checkers. - Usage:: + Protocol classes are defined like this:: - class Point2D(TypedDict): - x: int - y: int - label: str + class Proto(Protocol): + def meth(self) -> int: + ... - a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK - b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + Such classes are primarily used with static type checkers that recognize + structural subtyping (static duck-typing), for example:: - assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + class C: + def meth(self) -> int: + return 0 - To allow using this feature with older versions of Python that do not - support :pep:`526`, ``TypedDict`` supports two additional equivalent - syntactic forms: + def func(x: Proto) -> int: + return x.meth() - * Using a literal :class:`dict` as the second argument:: + func(C()) # Passes static type check - Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) + See :pep:`544` for more details. Protocol classes decorated with + :func:`runtime_checkable` (described later) act as simple-minded runtime + protocols that check only the presence of given attributes, ignoring their + type signatures. - * Using keyword arguments:: + Protocol classes can be generic, for example:: - Point2D = TypedDict('Point2D', x=int, y=int, label=str) + class GenProto[T](Protocol): + def meth(self) -> T: + ... - .. deprecated-removed:: 3.11 3.13 - The keyword-argument syntax is deprecated in 3.11 and will be removed - in 3.13. It may also be unsupported by static type checkers. + In code that needs to be compatible with Python 3.11 or older, generic + Protocols can be written as follows:: - The functional syntax should also be used when any of the keys are not valid - :ref:`identifiers `, for example because they are keywords or contain hyphens. - Example:: + T = TypeVar("T") - # raises SyntaxError - class Point2D(TypedDict): - in: int # 'in' is a keyword - x-y: int # name with hyphens + class GenProto(Protocol[T]): + def meth(self) -> T: + ... - # OK, functional syntax - Point2D = TypedDict('Point2D', {'in': int, 'x-y': int}) + .. versionadded:: 3.8 - By default, all keys must be present in a ``TypedDict``. It is possible to - mark individual keys as non-required using :data:`NotRequired`:: +.. decorator:: runtime_checkable - class Point2D(TypedDict): - x: int - y: int - label: NotRequired[str] + Mark a protocol class as a runtime protocol. - # Alternative syntax - Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': NotRequired[str]}) + Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. + This raises :exc:`TypeError` when applied to a non-protocol class. This + allows a simple-minded structural check, very similar to "one trick ponies" + in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: + + @runtime_checkable + class Closable(Protocol): + def close(self): ... + + assert isinstance(open('/some/file'), Closable) + + @runtime_checkable + class Named(Protocol): + name: str + + import threading + assert isinstance(threading.Thread(name='Bob'), Named) + + .. note:: + + :func:`!runtime_checkable` will check only the presence of the required + methods or attributes, not their type signatures or types. + For example, :class:`ssl.SSLObject` + is a class, therefore it passes an :func:`issubclass` + check against :ref:`Callable `. However, the + ``ssl.SSLObject.__init__`` method exists only to raise a + :exc:`TypeError` with a more informative message, therefore making + it impossible to call (instantiate) :class:`ssl.SSLObject`. + + .. note:: + + An :func:`isinstance` check against a runtime-checkable protocol can be + surprisingly slow compared to an ``isinstance()`` check against + a non-protocol class. Consider using alternative idioms such as + :func:`hasattr` calls for structural checks in performance-sensitive + code. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.12 + The internal implementation of :func:`isinstance` checks against + runtime-checkable protocols now uses :func:`inspect.getattr_static` + to look up attributes (previously, :func:`hasattr` was used). + As a result, some objects which used to be considered instances + of a runtime-checkable protocol may no longer be considered instances + of that protocol on Python 3.12+, and vice versa. + Most users are unlikely to be affected by this change. + + .. versionchanged:: 3.12 + The members of a runtime-checkable protocol are now considered "frozen" + at runtime as soon as the class has been created. Monkey-patching + attributes onto a runtime-checkable protocol will still work, but will + have no impact on :func:`isinstance` checks comparing objects to the + protocol. See :ref:`"What's new in Python 3.12" ` + for more details. + + +.. class:: TypedDict(dict) + + Special construct to add type hints to a dictionary. + At runtime it is a plain :class:`dict`. + + ``TypedDict`` declares a dictionary type that expects all of its + instances to have a certain set of keys, where each key is + associated with a value of a consistent type. This expectation + is not checked at runtime but is only enforced by type checkers. + Usage:: + + class Point2D(TypedDict): + x: int + y: int + label: str + + a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + + assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + + An alternative way to create a ``TypedDict`` is by using + function-call syntax. The second argument must be a literal :class:`dict`:: + + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) + + This functional syntax allows defining keys which are not valid + :ref:`identifiers `, for example because they are + keywords or contain hyphens:: + + # raises SyntaxError + class Point2D(TypedDict): + in: int # 'in' is a keyword + x-y: int # name with hyphens + + # OK, functional syntax + Point2D = TypedDict('Point2D', {'in': int, 'x-y': int}) + + By default, all keys must be present in a ``TypedDict``. It is possible to + mark individual keys as non-required using :data:`NotRequired`:: + + class Point2D(TypedDict): + x: int + y: int + label: NotRequired[str] + + # Alternative syntax + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': NotRequired[str]}) This means that a ``Point2D`` ``TypedDict`` can have the ``label`` key omitted. @@ -1870,11 +2366,19 @@ These are not used in annotations. They are building blocks for declaring types. class XZ(X, Z): pass # raises TypeError - T = TypeVar('T') - class XT(X, Generic[T]): pass # raises TypeError - A ``TypedDict`` can be generic:: + class Group[T](TypedDict): + key: T + group: list[T] + + To create a generic ``TypedDict`` that is compatible with Python 3.11 + or lower, inherit from :class:`Generic` explicitly: + + .. testcode:: + + T = TypeVar("T") + class Group(TypedDict, Generic[T]): key: T group: list[T] @@ -1886,7 +2390,9 @@ These are not used in annotations. They are building blocks for declaring types. .. attribute:: __total__ ``Point2D.__total__`` gives the value of the ``total`` argument. - Example:: + Example: + + .. doctest:: >>> from typing import TypedDict >>> class Point2D(TypedDict): pass @@ -1899,6 +2405,13 @@ These are not used in annotations. They are building blocks for declaring types. >>> Point3D.__total__ True + This attribute reflects *only* the value of the ``total`` argument + to the current ``TypedDict`` class, not whether the class is semantically + total. For example, a ``TypedDict`` with ``__total__`` set to True may + have keys marked with :data:`NotRequired`, or it may inherit from another + ``TypedDict`` with ``total=False``. Therefore, it is generally better to use + :attr:`__required_keys__` and :attr:`__optional_keys__` for introspection. + .. attribute:: __required_keys__ .. versionadded:: 3.9 @@ -1916,7 +2429,9 @@ These are not used in annotations. They are building blocks for declaring types. non-required keys in the same ``TypedDict`` . This is done by declaring a ``TypedDict`` with one value for the ``total`` argument and then inheriting from it in another ``TypedDict`` with a different value for - ``total``:: + ``total``: + + .. doctest:: >>> class Point2D(TypedDict, total=False): ... x: int @@ -1932,6 +2447,14 @@ These are not used in annotations. They are building blocks for declaring types. .. versionadded:: 3.9 + .. note:: + + If ``from __future__ import annotations`` is used or if annotations + are given as strings, annotations are not evaluated when the + ``TypedDict`` is defined. Therefore, the runtime introspection that + ``__required_keys__`` and ``__optional_keys__`` rely on may not work + properly, and the values of the attributes may be incorrect. + See :pep:`589` for more examples and detailed rules of using ``TypedDict``. .. versionadded:: 3.8 @@ -1943,1034 +2466,1241 @@ These are not used in annotations. They are building blocks for declaring types. .. versionchanged:: 3.11 Added support for generic ``TypedDict``\ s. -Generic concrete collections ----------------------------- + .. versionchanged:: 3.13 + Removed support for the keyword-argument method of creating ``TypedDict``\ s. -Corresponding to built-in types -""""""""""""""""""""""""""""""" - -.. class:: Dict(dict, MutableMapping[KT, VT]) + .. deprecated-removed:: 3.13 3.15 + When using the functional syntax to create a TypedDict class, failing to + pass a value to the 'fields' parameter (``TD = TypedDict("TD")``) is + deprecated. Passing ``None`` to the 'fields' parameter + (``TD = TypedDict("TD", None)``) is also deprecated. Both will be + disallowed in Python 3.15. To create a TypedDict class with 0 fields, + use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. - A generic version of :class:`dict`. - Useful for annotating return types. To annotate arguments it is preferred - to use an abstract collection type such as :class:`Mapping`. +Protocols +--------- - This type can be used as follows:: +The following protocols are provided by the typing module. All are decorated +with :func:`@runtime_checkable `. - def count_words(text: str) -> Dict[str, int]: - ... +.. class:: SupportsAbs - .. deprecated:: 3.9 - :class:`builtins.dict ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + An ABC with one abstract method ``__abs__`` that is covariant + in its return type. -.. class:: List(list, MutableSequence[T]) +.. class:: SupportsBytes - Generic version of :class:`list`. - Useful for annotating return types. To annotate arguments it is preferred - to use an abstract collection type such as :class:`Sequence` or - :class:`Iterable`. + An ABC with one abstract method ``__bytes__``. - This type may be used as follows:: +.. class:: SupportsComplex - T = TypeVar('T', int, float) + An ABC with one abstract method ``__complex__``. - def vec2(x: T, y: T) -> List[T]: - return [x, y] +.. class:: SupportsFloat - def keep_positives(vector: Sequence[T]) -> List[T]: - return [item for item in vector if item > 0] + An ABC with one abstract method ``__float__``. - .. deprecated:: 3.9 - :class:`builtins.list ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. class:: SupportsIndex -.. class:: Set(set, MutableSet[T]) + An ABC with one abstract method ``__index__``. - A generic version of :class:`builtins.set `. - Useful for annotating return types. To annotate arguments it is preferred - to use an abstract collection type such as :class:`AbstractSet`. + .. versionadded:: 3.8 - .. deprecated:: 3.9 - :class:`builtins.set ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. class:: SupportsInt -.. class:: FrozenSet(frozenset, AbstractSet[T_co]) + An ABC with one abstract method ``__int__``. - A generic version of :class:`builtins.frozenset `. +.. class:: SupportsRound - .. deprecated:: 3.9 - :class:`builtins.frozenset ` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + An ABC with one abstract method ``__round__`` + that is covariant in its return type. -.. note:: :data:`Tuple` is a special form. +ABCs for working with IO +------------------------ -Corresponding to types in :mod:`collections` -"""""""""""""""""""""""""""""""""""""""""""" +.. class:: IO + TextIO + BinaryIO -.. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) + Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` + and ``BinaryIO(IO[bytes])`` + represent the types of I/O streams such as returned by + :func:`open`. - A generic version of :class:`collections.defaultdict`. +Functions and decorators +------------------------ - .. versionadded:: 3.5.2 +.. function:: cast(typ, val) - .. deprecated:: 3.9 - :class:`collections.defaultdict` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Cast a value to a type. -.. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) + This returns the value unchanged. To the type checker this + signals that the return value has the designated type, but at + runtime we intentionally don't check anything (we want this + to be as fast as possible). - A generic version of :class:`collections.OrderedDict`. +.. function:: assert_type(val, typ, /) - .. versionadded:: 3.7.2 + Ask a static type checker to confirm that *val* has an inferred type of *typ*. - .. deprecated:: 3.9 - :class:`collections.OrderedDict` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. -.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + When a static type checker encounters a call to ``assert_type()``, it + emits an error if the value is not of the specified type:: - A generic version of :class:`collections.ChainMap`. + def greet(name: str) -> None: + assert_type(name, str) # OK, inferred type of `name` is `str` + assert_type(name, int) # type checker error - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 + This function is useful for ensuring the type checker's understanding of a + script is in line with the developer's intentions:: - .. deprecated:: 3.9 - :class:`collections.ChainMap` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + def complex_function(arg: object): + # Do some complex type-narrowing logic, + # after which we hope the inferred type will be `int` + ... + # Test whether the type checker correctly understands our function + assert_type(arg, int) -.. class:: Counter(collections.Counter, Dict[T, int]) + .. versionadded:: 3.11 - A generic version of :class:`collections.Counter`. +.. function:: assert_never(arg, /) - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 + Ask a static type checker to confirm that a line of code is unreachable. - .. deprecated:: 3.9 - :class:`collections.Counter` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Example:: -.. class:: Deque(deque, MutableSequence[T]) + def int_or_str(arg: int | str) -> None: + match arg: + case int(): + print("It's an int") + case str(): + print("It's a str") + case _ as unreachable: + assert_never(unreachable) - A generic version of :class:`collections.deque`. + Here, the annotations allow the type checker to infer that the + last case can never execute, because ``arg`` is either + an :class:`int` or a :class:`str`, and both options are covered by + earlier cases. - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 + If a type checker finds that a call to ``assert_never()`` is + reachable, it will emit an error. For example, if the type annotation + for ``arg`` was instead ``int | str | float``, the type checker would + emit an error pointing out that ``unreachable`` is of type :class:`float`. + For a call to ``assert_never`` to pass type checking, the inferred type of + the argument passed in must be the bottom type, :data:`Never`, and nothing + else. - .. deprecated:: 3.9 - :class:`collections.deque` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + At runtime, this throws an exception when called. -Other concrete types -"""""""""""""""""""" + .. seealso:: + `Unreachable Code and Exhaustiveness Checking + `__ has more + information about exhaustiveness checking with static typing. -.. class:: IO - TextIO - BinaryIO + .. versionadded:: 3.11 - Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` - and ``BinaryIO(IO[bytes])`` - represent the types of I/O streams such as returned by - :func:`open`. +.. function:: reveal_type(obj, /) - .. deprecated-removed:: 3.8 3.13 - The ``typing.io`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. + Reveal the inferred static type of an expression. -.. class:: Pattern - Match + When a static type checker encounters a call to this function, + it emits a diagnostic with the type of the argument. For example:: - These type aliases - correspond to the return types from :func:`re.compile` and - :func:`re.match`. These types (and the corresponding functions) - are generic in ``AnyStr`` and can be made specific by writing - ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or - ``Match[bytes]``. + x: int = 1 + reveal_type(x) # Revealed type is "builtins.int" - .. deprecated-removed:: 3.8 3.13 - The ``typing.re`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. + This can be useful when you want to debug how your type checker + handles a particular piece of code. - .. deprecated:: 3.9 - Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. - See :pep:`585` and :ref:`types-genericalias`. + The function returns its argument unchanged, which allows using + it within an expression:: -.. class:: Text + x = reveal_type(1) # Revealed type is "builtins.int" - ``Text`` is an alias for ``str``. It is provided to supply a forward - compatible path for Python 2 code: in Python 2, ``Text`` is an alias for - ``unicode``. + Most type checkers support ``reveal_type()`` anywhere, even if the + name is not imported from ``typing``. Importing the name from + ``typing`` allows your code to run without runtime errors and + communicates intent more clearly. - Use ``Text`` to indicate that a value must contain a unicode string in - a manner that is compatible with both Python 2 and Python 3:: + At runtime, this function prints the runtime type of its argument to stderr + and returns it unchanged:: - def add_unicode_checkmark(text: Text) -> Text: - return text + u' \u2713' + x = reveal_type(1) # prints "Runtime type is int" + print(x) # prints "1" - .. versionadded:: 3.5.2 + .. versionadded:: 3.11 - .. deprecated:: 3.11 - Python 2 is no longer supported, and most type checkers also no longer - support type checking Python 2 code. Removal of the alias is not - currently planned, but users are encouraged to use - :class:`str` instead of ``Text``. +.. decorator:: dataclass_transform(*, eq_default=True, order_default=False, \ + kw_only_default=False, frozen_default=False, \ + field_specifiers=(), **kwargs) -Abstract Base Classes ---------------------- + Decorator to mark an object as providing + :func:`dataclass `-like behavior. -Corresponding to collections in :mod:`collections.abc` -"""""""""""""""""""""""""""""""""""""""""""""""""""""" + ``dataclass_transform`` may be used to + decorate a class, metaclass, or a function that is itself a decorator. + The presence of ``@dataclass_transform()`` tells a static type checker that the + decorated object performs runtime "magic" that + transforms a class in a similar way to + :func:`@dataclasses.dataclass `. -.. class:: AbstractSet(Collection[T_co]) + Example usage with a decorator function: - A generic version of :class:`collections.abc.Set`. + .. testcode:: - .. deprecated:: 3.9 - :class:`collections.abc.Set` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + @dataclass_transform() + def create_model[T](cls: type[T]) -> type[T]: + ... + return cls -.. class:: ByteString(Sequence[int]) + @create_model + class CustomerModel: + id: int + name: str - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview` of byte sequences. + On a base class:: - .. deprecated-removed:: 3.9 3.14 - Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. + @dataclass_transform() + class ModelBase: ... -.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) + class CustomerModel(ModelBase): + id: int + name: str - A generic version of :class:`collections.abc.Collection` + On a metaclass:: - .. versionadded:: 3.6.0 + @dataclass_transform() + class ModelMeta(type): ... - .. deprecated:: 3.9 - :class:`collections.abc.Collection` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + class ModelBase(metaclass=ModelMeta): ... -.. class:: Container(Generic[T_co]) + class CustomerModel(ModelBase): + id: int + name: str - A generic version of :class:`collections.abc.Container`. + The ``CustomerModel`` classes defined above will + be treated by type checkers similarly to classes created with + :func:`@dataclasses.dataclass `. + For example, type checkers will assume these classes have + ``__init__`` methods that accept ``id`` and ``name``. - .. deprecated:: 3.9 - :class:`collections.abc.Container` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + The decorated class, metaclass, or function may accept the following bool + arguments which type checkers will assume have the same effect as they + would have on the + :func:`@dataclasses.dataclass` decorator: ``init``, + ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, + ``kw_only``, and ``slots``. It must be possible for the value of these + arguments (``True`` or ``False``) to be statically evaluated. -.. class:: ItemsView(MappingView, AbstractSet[tuple[KT_co, VT_co]]) + The arguments to the ``dataclass_transform`` decorator can be used to + customize the default behaviors of the decorated class, metaclass, or + function: - A generic version of :class:`collections.abc.ItemsView`. + :param bool eq_default: + Indicates whether the ``eq`` parameter is assumed to be + ``True`` or ``False`` if it is omitted by the caller. + Defaults to ``True``. - .. deprecated:: 3.9 - :class:`collections.abc.ItemsView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + :param bool order_default: + Indicates whether the ``order`` parameter is + assumed to be ``True`` or ``False`` if it is omitted by the caller. + Defaults to ``False``. -.. class:: KeysView(MappingView, AbstractSet[KT_co]) + :param bool kw_only_default: + Indicates whether the ``kw_only`` parameter is + assumed to be ``True`` or ``False`` if it is omitted by the caller. + Defaults to ``False``. - A generic version of :class:`collections.abc.KeysView`. + :param bool frozen_default: + Indicates whether the ``frozen`` parameter is + assumed to be ``True`` or ``False`` if it is omitted by the caller. + Defaults to ``False``. - .. deprecated:: 3.9 - :class:`collections.abc.KeysView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. versionadded:: 3.12 -.. class:: Mapping(Collection[KT], Generic[KT, VT_co]) + :param field_specifiers: + Specifies a static list of supported classes + or functions that describe fields, similar to :func:`dataclasses.field`. + Defaults to ``()``. + :type field_specifiers: tuple[Callable[..., Any], ...] - A generic version of :class:`collections.abc.Mapping`. - This type can be used as follows:: + :param Any \**kwargs: + Arbitrary other keyword arguments are accepted in order to allow for + possible future extensions. - def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: - return word_list[word] + Type checkers recognize the following optional parameters on field + specifiers: - .. deprecated:: 3.9 - :class:`collections.abc.Mapping` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. list-table:: **Recognised parameters for field specifiers** + :header-rows: 1 + :widths: 20 80 + + * - Parameter name + - Description + * - ``init`` + - Indicates whether the field should be included in the + synthesized ``__init__`` method. If unspecified, ``init`` defaults to + ``True``. + * - ``default`` + - Provides the default value for the field. + * - ``default_factory`` + - Provides a runtime callback that returns the + default value for the field. If neither ``default`` nor + ``default_factory`` are specified, the field is assumed to have no + default value and must be provided a value when the class is + instantiated. + * - ``factory`` + - An alias for the ``default_factory`` parameter on field specifiers. + * - ``kw_only`` + - Indicates whether the field should be marked as + keyword-only. If ``True``, the field will be keyword-only. If + ``False``, it will not be keyword-only. If unspecified, the value of + the ``kw_only`` parameter on the object decorated with + ``dataclass_transform`` will be used, or if that is unspecified, the + value of ``kw_only_default`` on ``dataclass_transform`` will be used. + * - ``alias`` + - Provides an alternative name for the field. This alternative + name is used in the synthesized ``__init__`` method. -.. class:: MappingView(Sized) + At runtime, this decorator records its arguments in the + ``__dataclass_transform__`` attribute on the decorated object. + It has no other runtime effect. - A generic version of :class:`collections.abc.MappingView`. + See :pep:`681` for more details. - .. deprecated:: 3.9 - :class:`collections.abc.MappingView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. versionadded:: 3.11 -.. class:: MutableMapping(Mapping[KT, VT]) +.. _overload: - A generic version of :class:`collections.abc.MutableMapping`. +.. decorator:: overload - .. deprecated:: 3.9 - :class:`collections.abc.MutableMapping` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Decorator for creating overloaded functions and methods. -.. class:: MutableSequence(Sequence[T]) + The ``@overload`` decorator allows describing functions and methods + that support multiple different combinations of argument types. A series + of ``@overload``-decorated definitions must be followed by exactly one + non-``@overload``-decorated definition (for the same function/method). - A generic version of :class:`collections.abc.MutableSequence`. + ``@overload``-decorated definitions are for the benefit of the + type checker only, since they will be overwritten by the + non-``@overload``-decorated definition. The non-``@overload``-decorated + definition, meanwhile, will be used at + runtime but should be ignored by a type checker. At runtime, calling + an ``@overload``-decorated function directly will raise + :exc:`NotImplementedError`. - .. deprecated:: 3.9 - :class:`collections.abc.MutableSequence` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + An example of overload that gives a more + precise type than can be expressed using a union or a type variable: -.. class:: MutableSet(AbstractSet[T]) + .. testcode:: - A generic version of :class:`collections.abc.MutableSet`. + @overload + def process(response: None) -> None: + ... + @overload + def process(response: int) -> tuple[int, str]: + ... + @overload + def process(response: bytes) -> str: + ... + def process(response): + ... # actual implementation goes here - .. deprecated:: 3.9 - :class:`collections.abc.MutableSet` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + See :pep:`484` for more details and comparison with other typing semantics. -.. class:: Sequence(Reversible[T_co], Collection[T_co]) + .. versionchanged:: 3.11 + Overloaded functions can now be introspected at runtime using + :func:`get_overloads`. - A generic version of :class:`collections.abc.Sequence`. - .. deprecated:: 3.9 - :class:`collections.abc.Sequence` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. function:: get_overloads(func) -.. class:: ValuesView(MappingView, Collection[_VT_co]) + Return a sequence of :func:`@overload `-decorated definitions for + *func*. - A generic version of :class:`collections.abc.ValuesView`. + *func* is the function object for the implementation of the + overloaded function. For example, given the definition of ``process`` in + the documentation for :func:`@overload `, + ``get_overloads(process)`` will return a sequence of three function objects + for the three defined overloads. If called on a function with no overloads, + ``get_overloads()`` returns an empty sequence. - .. deprecated:: 3.9 - :class:`collections.abc.ValuesView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + ``get_overloads()`` can be used for introspecting an overloaded function at + runtime. -Corresponding to other types in :mod:`collections.abc` -"""""""""""""""""""""""""""""""""""""""""""""""""""""" + .. versionadded:: 3.11 -.. class:: Iterable(Generic[T_co]) - A generic version of :class:`collections.abc.Iterable`. +.. function:: clear_overloads() - .. deprecated:: 3.9 - :class:`collections.abc.Iterable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Clear all registered overloads in the internal registry. -.. class:: Iterator(Iterable[T_co]) + This can be used to reclaim the memory used by the registry. - A generic version of :class:`collections.abc.Iterator`. + .. versionadded:: 3.11 - .. deprecated:: 3.9 - :class:`collections.abc.Iterator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. -.. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) +.. decorator:: final - A generator can be annotated by the generic type - ``Generator[YieldType, SendType, ReturnType]``. For example:: + Decorator to indicate final methods and final classes. - def echo_round() -> Generator[int, float, str]: - sent = yield 0 - while sent >= 0: - sent = yield round(sent) - return 'Done' + Decorating a method with ``@final`` indicates to a type checker that the + method cannot be overridden in a subclass. Decorating a class with ``@final`` + indicates that it cannot be subclassed. - Note that unlike many other generics in the typing module, the ``SendType`` - of :class:`Generator` behaves contravariantly, not covariantly or - invariantly. + For example:: - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker + ... - def infinite_stream(start: int) -> Generator[int, None, None]: - while True: - yield start - start += 1 + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... - Alternatively, annotate your generator as having a return type of - either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: + There is no runtime checking of these properties. See :pep:`591` for + more details. - def infinite_stream(start: int) -> Iterator[int]: - while True: - yield start - start += 1 + .. versionadded:: 3.8 - .. deprecated:: 3.9 - :class:`collections.abc.Generator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.11 + The decorator will now attempt to set a ``__final__`` attribute to ``True`` + on the decorated object. Thus, a check like + ``if getattr(obj, "__final__", False)`` can be used at runtime + to determine whether an object ``obj`` has been marked as final. + If the decorated object does not support setting attributes, + the decorator returns the object unchanged without raising an exception. -.. class:: Hashable - An alias to :class:`collections.abc.Hashable`. +.. decorator:: no_type_check - .. deprecated:: 3.12 - Use :class:`collections.abc.Hashable` directly instead. + Decorator to indicate that annotations are not type hints. -.. class:: Reversible(Iterable[T_co]) + This works as a class or function :term:`decorator`. With a class, it + applies recursively to all methods and classes defined in that class + (but not to methods defined in its superclasses or subclasses). Type + checkers will ignore all annotations in a function or class with this + decorator. - A generic version of :class:`collections.abc.Reversible`. + ``@no_type_check`` mutates the decorated object in place. - .. deprecated:: 3.9 - :class:`collections.abc.Reversible` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +.. decorator:: no_type_check_decorator -.. class:: Sized + Decorator to give another decorator the :func:`no_type_check` effect. - An alias to :class:`collections.abc.Sized`. + This wraps the decorator with something that wraps the decorated + function in :func:`no_type_check`. - .. deprecated:: 3.12 - Use :class:`collections.abc.Sized` directly instead. + .. deprecated-removed:: 3.13 3.15 + No type checker ever added support for ``@no_type_check_decorator``. It + is therefore deprecated, and will be removed in Python 3.15. -Asynchronous programming -"""""""""""""""""""""""" +.. decorator:: override -.. class:: Coroutine(Awaitable[V_co], Generic[T_co, T_contra, V_co]) + Decorator to indicate that a method in a subclass is intended to override a + method or attribute in a superclass. - A generic version of :class:`collections.abc.Coroutine`. - The variance and order of type variables - correspond to those of :class:`Generator`, for example:: + Type checkers should emit an error if a method decorated with ``@override`` + does not, in fact, override anything. + This helps prevent bugs that may occur when a base class is changed without + an equivalent change to a child class. - from collections.abc import Coroutine - c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere - x = c.send('hi') # Inferred type of 'x' is list[str] - async def bar() -> None: - y = await c # Inferred type of 'y' is int + For example: - .. versionadded:: 3.5.3 + .. testcode:: - .. deprecated:: 3.9 - :class:`collections.abc.Coroutine` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + class Base: + def log_status(self) -> None: + ... -.. class:: AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra]) + class Sub(Base): + @override + def log_status(self) -> None: # Okay: overrides Base.log_status + ... - An async generator can be annotated by the generic type - ``AsyncGenerator[YieldType, SendType]``. For example:: + @override + def done(self) -> None: # Error reported by type checker + ... - async def echo_round() -> AsyncGenerator[int, float]: - sent = yield 0 - while sent >= 0.0: - rounded = await round(sent) - sent = yield rounded + There is no runtime checking of this property. - Unlike normal generators, async generators cannot return a value, so there - is no ``ReturnType`` type parameter. As with :class:`Generator`, the - ``SendType`` behaves contravariantly. + The decorator will attempt to set an ``__override__`` attribute to ``True`` on + the decorated object. Thus, a check like + ``if getattr(obj, "__override__", False)`` can be used at runtime to determine + whether an object ``obj`` has been marked as an override. If the decorated object + does not support setting attributes, the decorator returns the object unchanged + without raising an exception. - If your generator will only yield values, set the ``SendType`` to - ``None``:: + See :pep:`698` for more details. - async def infinite_stream(start: int) -> AsyncGenerator[int, None]: - while True: - yield start - start = await increment(start) + .. versionadded:: 3.12 - Alternatively, annotate your generator as having a return type of - either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: - async def infinite_stream(start: int) -> AsyncIterator[int]: - while True: - yield start - start = await increment(start) +.. decorator:: type_check_only - .. versionadded:: 3.6.1 + Decorator to mark a class or function as unavailable at runtime. - .. deprecated:: 3.9 - :class:`collections.abc.AsyncGenerator` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + This decorator is itself not available at runtime. It is mainly + intended to mark classes that are defined in type stub files if + an implementation returns an instance of a private class:: -.. class:: AsyncIterable(Generic[T_co]) + @type_check_only + class Response: # private or not available at runtime + code: int + def get_header(self, name: str) -> str: ... - A generic version of :class:`collections.abc.AsyncIterable`. + def fetch_response() -> Response: ... - .. versionadded:: 3.5.2 + Note that returning instances of private classes is not recommended. + It is usually preferable to make such classes public. - .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. +Introspection helpers +--------------------- -.. class:: AsyncIterator(AsyncIterable[T_co]) +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) - A generic version of :class:`collections.abc.AsyncIterator`. + Return a dictionary containing type hints for a function, method, module + or class object. - .. versionadded:: 3.5.2 + This is often the same as ``obj.__annotations__``. In addition, + forward references encoded as string literals are handled by evaluating + them in ``globals`` and ``locals`` namespaces. For a class ``C``, return + a dictionary constructed by merging all the ``__annotations__`` along + ``C.__mro__`` in reverse order. - .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + The function recursively replaces all ``Annotated[T, ...]`` with ``T``, + unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for + more information). For example: -.. class:: Awaitable(Generic[T_co]) + .. testcode:: - A generic version of :class:`collections.abc.Awaitable`. + class Student(NamedTuple): + name: Annotated[str, 'some marker'] - .. versionadded:: 3.5.2 + assert get_type_hints(Student) == {'name': str} + assert get_type_hints(Student, include_extras=False) == {'name': str} + assert get_type_hints(Student, include_extras=True) == { + 'name': Annotated[str, 'some marker'] + } - .. deprecated:: 3.9 - :class:`collections.abc.Awaitable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. note:: + :func:`get_type_hints` does not work with imported + :ref:`type aliases ` that include forward references. + Enabling postponed evaluation of annotations (:pep:`563`) may remove + the need for most forward references. -Context manager types -""""""""""""""""""""" + .. versionchanged:: 3.9 + Added ``include_extras`` parameter as part of :pep:`593`. + See the documentation on :data:`Annotated` for more information. -.. class:: ContextManager(Generic[T_co]) + .. versionchanged:: 3.11 + Previously, ``Optional[t]`` was added for function and method annotations + if a default value equal to ``None`` was set. + Now the annotation is returned unchanged. - A generic version of :class:`contextlib.AbstractContextManager`. +.. function:: get_origin(tp) - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.0 + Get the unsubscripted version of a type: for a typing object of the form + ``X[Y, Z, ...]`` return ``X``. - .. deprecated:: 3.9 - :class:`contextlib.AbstractContextManager` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + If ``X`` is a typing-module alias for a builtin or + :mod:`collections` class, it will be normalized to the original class. + If ``X`` is an instance of :class:`ParamSpecArgs` or :class:`ParamSpecKwargs`, + return the underlying :class:`ParamSpec`. + Return ``None`` for unsupported objects. -.. class:: AsyncContextManager(Generic[T_co]) + Examples: - A generic version of :class:`contextlib.AbstractAsyncContextManager`. + .. testcode:: - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 + assert get_origin(str) is None + assert get_origin(Dict[str, int]) is dict + assert get_origin(Union[int, str]) is Union + P = ParamSpec('P') + assert get_origin(P.args) is P + assert get_origin(P.kwargs) is P - .. deprecated:: 3.9 - :class:`contextlib.AbstractAsyncContextManager` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + .. versionadded:: 3.8 -Protocols ---------- +.. function:: get_args(tp) -These protocols are decorated with :func:`runtime_checkable`. + Get type arguments with all substitutions performed: for a typing object + of the form ``X[Y, Z, ...]`` return ``(Y, Z, ...)``. -.. class:: SupportsAbs + If ``X`` is a union or :class:`Literal` contained in another + generic type, the order of ``(Y, Z, ...)`` may be different from the order + of the original arguments ``[Y, Z, ...]`` due to type caching. + Return ``()`` for unsupported objects. - An ABC with one abstract method ``__abs__`` that is covariant - in its return type. + Examples: -.. class:: SupportsBytes + .. testcode:: - An ABC with one abstract method ``__bytes__``. + assert get_args(int) == () + assert get_args(Dict[int, str]) == (int, str) + assert get_args(Union[int, str]) == (int, str) -.. class:: SupportsComplex + .. versionadded:: 3.8 - An ABC with one abstract method ``__complex__``. +.. function:: get_protocol_members(tp) -.. class:: SupportsFloat + Return the set of members defined in a :class:`Protocol`. - An ABC with one abstract method ``__float__``. + .. doctest:: -.. class:: SupportsIndex + >>> from typing import Protocol, get_protocol_members + >>> class P(Protocol): + ... def a(self) -> str: ... + ... b: int + >>> get_protocol_members(P) == frozenset({'a', 'b'}) + True - An ABC with one abstract method ``__index__``. + Raise :exc:`TypeError` for arguments that are not Protocols. - .. versionadded:: 3.8 + .. versionadded:: 3.13 -.. class:: SupportsInt +.. function:: is_protocol(tp) - An ABC with one abstract method ``__int__``. + Determine if a type is a :class:`Protocol`. -.. class:: SupportsRound + For example:: - An ABC with one abstract method ``__round__`` - that is covariant in its return type. + class P(Protocol): + def a(self) -> str: ... + b: int -Functions and decorators ------------------------- + is_protocol(P) # => True + is_protocol(int) # => False -.. function:: cast(typ, val) + .. versionadded:: 3.13 - Cast a value to a type. +.. function:: is_typeddict(tp) - This returns the value unchanged. To the type checker this - signals that the return value has the designated type, but at - runtime we intentionally don't check anything (we want this - to be as fast as possible). + Check if a type is a :class:`TypedDict`. -.. function:: assert_type(val, typ, /) + For example: - Ask a static type checker to confirm that *val* has an inferred type of *typ*. + .. testcode:: - At runtime this does nothing: it returns the first argument unchanged with no - checks or side effects, no matter the actual type of the argument. + class Film(TypedDict): + title: str + year: int - When a static type checker encounters a call to ``assert_type()``, it - emits an error if the value is not of the specified type:: + assert is_typeddict(Film) + assert not is_typeddict(list | str) - def greet(name: str) -> None: - assert_type(name, str) # OK, inferred type of `name` is `str` - assert_type(name, int) # type checker error + # TypedDict is a factory for creating typed dicts, + # not a typed dict itself + assert not is_typeddict(TypedDict) - This function is useful for ensuring the type checker's understanding of a - script is in line with the developer's intentions:: + .. versionadded:: 3.10 - def complex_function(arg: object): - # Do some complex type-narrowing logic, - # after which we hope the inferred type will be `int` - ... - # Test whether the type checker correctly understands our function - assert_type(arg, int) +.. class:: ForwardRef - .. versionadded:: 3.11 + Class used for internal typing representation of string forward references. -.. function:: assert_never(arg, /) + For example, ``List["SomeClass"]`` is implicitly transformed into + ``List[ForwardRef("SomeClass")]``. ``ForwardRef`` should not be instantiated by + a user, but may be used by introspection tools. - Ask a static type checker to confirm that a line of code is unreachable. + .. note:: + :pep:`585` generic types such as ``list["SomeClass"]`` will not be + implicitly transformed into ``list[ForwardRef("SomeClass")]`` and thus + will not automatically resolve to ``list[SomeClass]``. - Example:: + .. versionadded:: 3.7.4 - def int_or_str(arg: int | str) -> None: - match arg: - case int(): - print("It's an int") - case str(): - print("It's a str") - case _ as unreachable: - assert_never(unreachable) +Constant +-------- - Here, the annotations allow the type checker to infer that the - last case can never execute, because ``arg`` is either - an :class:`int` or a :class:`str`, and both options are covered by - earlier cases. - If a type checker finds that a call to ``assert_never()`` is - reachable, it will emit an error. For example, if the type annotation - for ``arg`` was instead ``int | str | float``, the type checker would - emit an error pointing out that ``unreachable`` is of type :class:`float`. - For a call to ``assert_never`` to pass type checking, the inferred type of - the argument passed in must be the bottom type, :data:`Never`, and nothing - else. +.. data:: TYPE_CHECKING - At runtime, this throws an exception when called. + A special constant that is assumed to be ``True`` by 3rd party static + type checkers. It is ``False`` at runtime. - .. seealso:: - `Unreachable Code and Exhaustiveness Checking - `__ has more - information about exhaustiveness checking with static typing. + Usage:: - .. versionadded:: 3.11 + if TYPE_CHECKING: + import expensive_mod -.. function:: reveal_type(obj, /) + def fun(arg: 'expensive_mod.SomeType') -> None: + local_var: expensive_mod.AnotherType = other_fun() - Reveal the inferred static type of an expression. + The first type annotation must be enclosed in quotes, making it a + "forward reference", to hide the ``expensive_mod`` reference from the + interpreter runtime. Type annotations for local variables are not + evaluated, so the second annotation does not need to be enclosed in quotes. - When a static type checker encounters a call to this function, - it emits a diagnostic with the type of the argument. For example:: + .. note:: - x: int = 1 - reveal_type(x) # Revealed type is "builtins.int" + If ``from __future__ import annotations`` is used, + annotations are not evaluated at function definition time. + Instead, they are stored as strings in ``__annotations__``. + This makes it unnecessary to use quotes around the annotation + (see :pep:`563`). - This can be useful when you want to debug how your type checker - handles a particular piece of code. + .. versionadded:: 3.5.2 - The function returns its argument unchanged, which allows using - it within an expression:: +.. _generic-concrete-collections: +.. _deprecated-aliases: - x = reveal_type(1) # Revealed type is "builtins.int" +Deprecated aliases +------------------ - Most type checkers support ``reveal_type()`` anywhere, even if the - name is not imported from ``typing``. Importing the name from - ``typing`` allows your code to run without runtime errors and - communicates intent more clearly. +This module defines several deprecated aliases to pre-existing +standard library classes. These were originally included in the typing +module in order to support parameterizing these generic classes using ``[]``. +However, the aliases became redundant in Python 3.9 when the +corresponding pre-existing classes were enhanced to support ``[]`` (see +:pep:`585`). - At runtime, this function prints the runtime type of its argument to stderr - and returns it unchanged:: +The redundant types are deprecated as of Python 3.9. However, while the aliases +may be removed at some point, removal of these aliases is not currently +planned. As such, no deprecation warnings are currently issued by the +interpreter for these aliases. - x = reveal_type(1) # prints "Runtime type is int" - print(x) # prints "1" +If at some point it is decided to remove these deprecated aliases, a +deprecation warning will be issued by the interpreter for at least two releases +prior to removal. The aliases are guaranteed to remain in the typing module +without deprecation warnings until at least Python 3.14. - .. versionadded:: 3.11 +Type checkers are encouraged to flag uses of the deprecated types if the +program they are checking targets a minimum Python version of 3.9 or newer. -.. decorator:: dataclass_transform +.. _corresponding-to-built-in-types: - :data:`~typing.dataclass_transform` may be used to - decorate a class, metaclass, or a function that is itself a decorator. - The presence of ``@dataclass_transform()`` tells a static type checker that the - decorated object performs runtime "magic" that - transforms a class, giving it :func:`dataclasses.dataclass`-like behaviors. +Aliases to built-in types +""""""""""""""""""""""""" - Example usage with a decorator function:: +.. class:: Dict(dict, MutableMapping[KT, VT]) - T = TypeVar("T") + Deprecated alias to :class:`dict`. - @dataclass_transform() - def create_model(cls: type[T]) -> type[T]: + Note that to annotate arguments, it is preferred + to use an abstract collection type such as :class:`Mapping` + rather than to use :class:`dict` or :class:`!typing.Dict`. + + This type can be used as follows:: + + def count_words(text: str) -> Dict[str, int]: ... - return cls - @create_model - class CustomerModel: - id: int - name: str + .. deprecated:: 3.9 + :class:`builtins.dict ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - On a base class:: +.. class:: List(list, MutableSequence[T]) - @dataclass_transform() - class ModelBase: ... + Deprecated alias to :class:`list`. - class CustomerModel(ModelBase): - id: int - name: str + Note that to annotate arguments, it is preferred + to use an abstract collection type such as :class:`Sequence` or + :class:`Iterable` rather than to use :class:`list` or :class:`!typing.List`. - On a metaclass:: + This type may be used as follows:: - @dataclass_transform() - class ModelMeta(type): ... + def vec2[T: (int, float)](x: T, y: T) -> List[T]: + return [x, y] - class ModelBase(metaclass=ModelMeta): ... + def keep_positives[T: (int, float)](vector: Sequence[T]) -> List[T]: + return [item for item in vector if item > 0] - class CustomerModel(ModelBase): - id: int - name: str + .. deprecated:: 3.9 + :class:`builtins.list ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - The ``CustomerModel`` classes defined above will - be treated by type checkers similarly to classes created with - :func:`@dataclasses.dataclass `. - For example, type checkers will assume these classes have - ``__init__`` methods that accept ``id`` and ``name``. +.. class:: Set(set, MutableSet[T]) - The decorated class, metaclass, or function may accept the following bool - arguments which type checkers will assume have the same effect as they - would have on the - :func:`@dataclasses.dataclass` decorator: ``init``, - ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, - ``kw_only``, and ``slots``. It must be possible for the value of these - arguments (``True`` or ``False``) to be statically evaluated. + Deprecated alias to :class:`builtins.set `. - The arguments to the ``dataclass_transform`` decorator can be used to - customize the default behaviors of the decorated class, metaclass, or - function: + Note that to annotate arguments, it is preferred + to use an abstract collection type such as :class:`AbstractSet` + rather than to use :class:`set` or :class:`!typing.Set`. - * ``eq_default`` indicates whether the ``eq`` parameter is assumed to be - ``True`` or ``False`` if it is omitted by the caller. - * ``order_default`` indicates whether the ``order`` parameter is - assumed to be True or False if it is omitted by the caller. - * ``kw_only_default`` indicates whether the ``kw_only`` parameter is - assumed to be True or False if it is omitted by the caller. - * ``frozen_default`` indicates whether the ``frozen`` parameter is - assumed to be True or False if it is omitted by the caller. - - .. versionadded:: 3.12 - * ``field_specifiers`` specifies a static list of supported classes - or functions that describe fields, similar to ``dataclasses.field()``. - * Arbitrary other keyword arguments are accepted in order to allow for - possible future extensions. - - Type checkers recognize the following optional arguments on field - specifiers: + .. deprecated:: 3.9 + :class:`builtins.set ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - * ``init`` indicates whether the field should be included in the - synthesized ``__init__`` method. If unspecified, ``init`` defaults to - ``True``. - * ``default`` provides the default value for the field. - * ``default_factory`` provides a runtime callback that returns the - default value for the field. If neither ``default`` nor - ``default_factory`` are specified, the field is assumed to have no - default value and must be provided a value when the class is - instantiated. - * ``factory`` is an alias for ``default_factory``. - * ``kw_only`` indicates whether the field should be marked as - keyword-only. If ``True``, the field will be keyword-only. If - ``False``, it will not be keyword-only. If unspecified, the value of - the ``kw_only`` parameter on the object decorated with - ``dataclass_transform`` will be used, or if that is unspecified, the - value of ``kw_only_default`` on ``dataclass_transform`` will be used. - * ``alias`` provides an alternative name for the field. This alternative - name is used in the synthesized ``__init__`` method. +.. class:: FrozenSet(frozenset, AbstractSet[T_co]) - At runtime, this decorator records its arguments in the - ``__dataclass_transform__`` attribute on the decorated object. - It has no other runtime effect. + Deprecated alias to :class:`builtins.frozenset `. - See :pep:`681` for more details. + .. deprecated:: 3.9 + :class:`builtins.frozenset ` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. versionadded:: 3.11 +.. data:: Tuple -.. decorator:: overload + Deprecated alias for :class:`tuple`. - The ``@overload`` decorator allows describing functions and methods - that support multiple different combinations of argument types. A series - of ``@overload``-decorated definitions must be followed by exactly one - non-``@overload``-decorated definition (for the same function/method). - The ``@overload``-decorated definitions are for the benefit of the - type checker only, since they will be overwritten by the - non-``@overload``-decorated definition, while the latter is used at - runtime but should be ignored by a type checker. At runtime, calling - a ``@overload``-decorated function directly will raise - :exc:`NotImplementedError`. An example of overload that gives a more - precise type than can be expressed using a union or a type variable:: + :class:`tuple` and ``Tuple`` are special-cased in the type system; see + :ref:`annotating-tuples` for more details. - @overload - def process(response: None) -> None: - ... - @overload - def process(response: int) -> tuple[int, str]: - ... - @overload - def process(response: bytes) -> str: - ... - def process(response): - + .. deprecated:: 3.9 + :class:`builtins.tuple ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - See :pep:`484` for more details and comparison with other typing semantics. +.. class:: Type(Generic[CT_co]) - .. versionchanged:: 3.11 - Overloaded functions can now be introspected at runtime using - :func:`get_overloads`. + Deprecated alias to :class:`type`. + See :ref:`type-of-class-objects` for details on using :class:`type` or + ``typing.Type`` in type annotations. -.. function:: get_overloads(func) + .. versionadded:: 3.5.2 - Return a sequence of :func:`@overload `-decorated definitions for - *func*. *func* is the function object for the implementation of the - overloaded function. For example, given the definition of ``process`` in - the documentation for :func:`@overload `, - ``get_overloads(process)`` will return a sequence of three function objects - for the three defined overloads. If called on a function with no overloads, - ``get_overloads()`` returns an empty sequence. + .. deprecated:: 3.9 + :class:`builtins.type ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - ``get_overloads()`` can be used for introspecting an overloaded function at - runtime. +.. _corresponding-to-types-in-collections: - .. versionadded:: 3.11 +Aliases to types in :mod:`collections` +"""""""""""""""""""""""""""""""""""""" +.. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) -.. function:: clear_overloads() + Deprecated alias to :class:`collections.defaultdict`. - Clear all registered overloads in the internal registry. This can be used - to reclaim the memory used by the registry. + .. versionadded:: 3.5.2 - .. versionadded:: 3.11 + .. deprecated:: 3.9 + :class:`collections.defaultdict` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. +.. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) -.. decorator:: final + Deprecated alias to :class:`collections.OrderedDict`. - A decorator to indicate to type checkers that the decorated method - cannot be overridden, and the decorated class cannot be subclassed. - For example:: + .. versionadded:: 3.7.2 - class Base: - @final - def done(self) -> None: - ... - class Sub(Base): - def done(self) -> None: # Error reported by type checker - ... + .. deprecated:: 3.9 + :class:`collections.OrderedDict` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - @final - class Leaf: - ... - class Other(Leaf): # Error reported by type checker - ... +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) - There is no runtime checking of these properties. See :pep:`591` for - more details. + Deprecated alias to :class:`collections.ChainMap`. - .. versionadded:: 3.8 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - .. versionchanged:: 3.11 - The decorator will now set the ``__final__`` attribute to ``True`` - on the decorated object. Thus, a check like - ``if getattr(obj, "__final__", False)`` can be used at runtime - to determine whether an object ``obj`` has been marked as final. - If the decorated object does not support setting attributes, - the decorator returns the object unchanged without raising an exception. + .. deprecated:: 3.9 + :class:`collections.ChainMap` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. +.. class:: Counter(collections.Counter, Dict[T, int]) -.. decorator:: no_type_check + Deprecated alias to :class:`collections.Counter`. - Decorator to indicate that annotations are not type hints. + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - This works as class or function :term:`decorator`. With a class, it - applies recursively to all methods and classes defined in that class - (but not to methods defined in its superclasses or subclasses). + .. deprecated:: 3.9 + :class:`collections.Counter` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - This mutates the function(s) in place. +.. class:: Deque(deque, MutableSequence[T]) -.. decorator:: no_type_check_decorator + Deprecated alias to :class:`collections.deque`. - Decorator to give another decorator the :func:`no_type_check` effect. + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - This wraps the decorator with something that wraps the decorated - function in :func:`no_type_check`. + .. deprecated:: 3.9 + :class:`collections.deque` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. +.. _other-concrete-types: -.. decorator:: override +Aliases to other concrete types +""""""""""""""""""""""""""""""" - A decorator for methods that indicates to type checkers that this method - should override a method or attribute with the same name on a base class. - This helps prevent bugs that may occur when a base class is changed without - an equivalent change to a child class. +.. class:: Pattern + Match - For example:: + Deprecated aliases corresponding to the return types from + :func:`re.compile` and :func:`re.match`. - class Base: - def log_status(self) + These types (and the corresponding functions) are generic over + :data:`AnyStr`. ``Pattern`` can be specialised as ``Pattern[str]`` or + ``Pattern[bytes]``; ``Match`` can be specialised as ``Match[str]`` or + ``Match[bytes]``. - class Sub(Base): - @override - def log_status(self) -> None: # Okay: overrides Base.log_status - ... + .. deprecated:: 3.9 + Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. + See :pep:`585` and :ref:`types-genericalias`. - @override - def done(self) -> None: # Error reported by type checker - ... +.. class:: Text - There is no runtime checking of this property. + Deprecated alias for :class:`str`. - The decorator will set the ``__override__`` attribute to ``True`` on - the decorated object. Thus, a check like - ``if getattr(obj, "__override__", False)`` can be used at runtime to determine - whether an object ``obj`` has been marked as an override. If the decorated object - does not support setting attributes, the decorator returns the object unchanged - without raising an exception. + ``Text`` is provided to supply a forward + compatible path for Python 2 code: in Python 2, ``Text`` is an alias for + ``unicode``. - See :pep:`698` for more details. + Use ``Text`` to indicate that a value must contain a unicode string in + a manner that is compatible with both Python 2 and Python 3:: - .. versionadded:: 3.12 + def add_unicode_checkmark(text: Text) -> Text: + return text + u' \u2713' + .. versionadded:: 3.5.2 -.. decorator:: type_check_only + .. deprecated:: 3.11 + Python 2 is no longer supported, and most type checkers also no longer + support type checking Python 2 code. Removal of the alias is not + currently planned, but users are encouraged to use + :class:`str` instead of ``Text``. - Decorator to mark a class or function to be unavailable at runtime. +.. _abstract-base-classes: +.. _corresponding-to-collections-in-collections-abc: - This decorator is itself not available at runtime. It is mainly - intended to mark classes that are defined in type stub files if - an implementation returns an instance of a private class:: +Aliases to container ABCs in :mod:`collections.abc` +""""""""""""""""""""""""""""""""""""""""""""""""""" - @type_check_only - class Response: # private or not available at runtime - code: int - def get_header(self, name: str) -> str: ... +.. class:: AbstractSet(Collection[T_co]) - def fetch_response() -> Response: ... + Deprecated alias to :class:`collections.abc.Set`. - Note that returning instances of private classes is not recommended. - It is usually preferable to make such classes public. + .. deprecated:: 3.9 + :class:`collections.abc.Set` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -Introspection helpers ---------------------- +.. class:: ByteString(Sequence[int]) -.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) + This type represents the types :class:`bytes`, :class:`bytearray`, + and :class:`memoryview` of byte sequences. - Return a dictionary containing type hints for a function, method, module - or class object. + .. deprecated-removed:: 3.9 3.14 + Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. +.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) - The function recursively replaces all ``Annotated[T, ...]`` with ``T``, - unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for - more information). For example:: + Deprecated alias to :class:`collections.abc.Collection`. - class Student(NamedTuple): - name: Annotated[str, 'some marker'] + .. versionadded:: 3.6.0 - get_type_hints(Student) == {'name': str} - get_type_hints(Student, include_extras=False) == {'name': str} - get_type_hints(Student, include_extras=True) == { - 'name': Annotated[str, 'some marker'] - } + .. deprecated:: 3.9 + :class:`collections.abc.Collection` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. note:: +.. class:: Container(Generic[T_co]) - :func:`get_type_hints` does not work with imported - :ref:`type aliases ` that include forward references. - Enabling postponed evaluation of annotations (:pep:`563`) may remove - the need for most forward references. + Deprecated alias to :class:`collections.abc.Container`. - .. versionchanged:: 3.9 - Added ``include_extras`` parameter as part of :pep:`593`. + .. deprecated:: 3.9 + :class:`collections.abc.Container` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. versionchanged:: 3.11 - Previously, ``Optional[t]`` was added for function and method annotations - if a default value equal to ``None`` was set. - Now the annotation is returned unchanged. +.. class:: ItemsView(MappingView, AbstractSet[tuple[KT_co, VT_co]]) -.. function:: get_origin(tp) + Deprecated alias to :class:`collections.abc.ItemsView`. - Get the unsubscripted version of a type: for a typing object of the form - ``X[Y, Z, ...]`` return ``X``. If ``X`` is a generic alias for a builtin or - :mod:`collections` class, it gets normalized to the original class. - If ``X`` is an instance of :class:`ParamSpecArgs` or :class:`ParamSpecKwargs`, - return the underlying :class:`ParamSpec`. - Return ``None`` for unsupported objects. - Examples:: + .. deprecated:: 3.9 + :class:`collections.abc.ItemsView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - assert get_origin(str) is None - assert get_origin(Dict[str, int]) is dict - assert get_origin(Union[int, str]) is Union - P = ParamSpec('P') - assert get_origin(P.args) is P - assert get_origin(P.kwargs) is P +.. class:: KeysView(MappingView, AbstractSet[KT_co]) - .. versionadded:: 3.8 + Deprecated alias to :class:`collections.abc.KeysView`. -.. function:: get_args(tp) + .. deprecated:: 3.9 + :class:`collections.abc.KeysView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - Get type arguments with all substitutions performed: for a typing object - of the form ``X[Y, Z, ...]`` return ``(Y, Z, ...)``. - If ``X`` is a union or :class:`Literal` contained in another - generic type, the order of ``(Y, Z, ...)`` may be different from the order - of the original arguments ``[Y, Z, ...]`` due to type caching. - Return ``()`` for unsupported objects. - Examples:: +.. class:: Mapping(Collection[KT], Generic[KT, VT_co]) - assert get_args(int) == () - assert get_args(Dict[int, str]) == (int, str) - assert get_args(Union[int, str]) == (int, str) + Deprecated alias to :class:`collections.abc.Mapping`. - .. versionadded:: 3.8 + This type can be used as follows:: -.. function:: is_typeddict(tp) + def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: + return word_list[word] - Check if a type is a :class:`TypedDict`. + .. deprecated:: 3.9 + :class:`collections.abc.Mapping` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - For example:: +.. class:: MappingView(Sized) - class Film(TypedDict): - title: str - year: int + Deprecated alias to :class:`collections.abc.MappingView`. - is_typeddict(Film) # => True - is_typeddict(list | str) # => False + .. deprecated:: 3.9 + :class:`collections.abc.MappingView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. versionadded:: 3.10 +.. class:: MutableMapping(Mapping[KT, VT]) -.. class:: ForwardRef + Deprecated alias to :class:`collections.abc.MutableMapping`. - A class used for internal typing representation of string forward references. - For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by - a user, but may be used by introspection tools. + .. deprecated:: 3.9 + :class:`collections.abc.MutableMapping` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - .. note:: - :pep:`585` generic types such as ``list["SomeClass"]`` will not be - implicitly transformed into ``list[ForwardRef("SomeClass")]`` and thus - will not automatically resolve to ``list[SomeClass]``. +.. class:: MutableSequence(Sequence[T]) - .. versionadded:: 3.7.4 + Deprecated alias to :class:`collections.abc.MutableSequence`. -Constant --------- + .. deprecated:: 3.9 + :class:`collections.abc.MutableSequence` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. data:: TYPE_CHECKING +.. class:: MutableSet(AbstractSet[T]) - A special constant that is assumed to be ``True`` by 3rd party static - type checkers. It is ``False`` at runtime. Usage:: + Deprecated alias to :class:`collections.abc.MutableSet`. - if TYPE_CHECKING: - import expensive_mod + .. deprecated:: 3.9 + :class:`collections.abc.MutableSet` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - def fun(arg: 'expensive_mod.SomeType') -> None: - local_var: expensive_mod.AnotherType = other_fun() +.. class:: Sequence(Reversible[T_co], Collection[T_co]) - The first type annotation must be enclosed in quotes, making it a - "forward reference", to hide the ``expensive_mod`` reference from the - interpreter runtime. Type annotations for local variables are not - evaluated, so the second annotation does not need to be enclosed in quotes. + Deprecated alias to :class:`collections.abc.Sequence`. - .. note:: + .. deprecated:: 3.9 + :class:`collections.abc.Sequence` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. - If ``from __future__ import annotations`` is used, - annotations are not evaluated at function definition time. - Instead, they are stored as strings in ``__annotations__``. - This makes it unnecessary to use quotes around the annotation - (see :pep:`563`). +.. class:: ValuesView(MappingView, Collection[_VT_co]) + + Deprecated alias to :class:`collections.abc.ValuesView`. + + .. deprecated:: 3.9 + :class:`collections.abc.ValuesView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. _asynchronous-programming: + +Aliases to asynchronous ABCs in :mod:`collections.abc` +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +.. class:: Coroutine(Awaitable[ReturnType], Generic[YieldType, SendType, ReturnType]) + + Deprecated alias to :class:`collections.abc.Coroutine`. + + The variance and order of type variables + correspond to those of :class:`Generator`, for example:: + + from collections.abc import Coroutine + c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere + x = c.send('hi') # Inferred type of 'x' is list[str] + async def bar() -> None: + y = await c # Inferred type of 'y' is int + + .. versionadded:: 3.5.3 + + .. deprecated:: 3.9 + :class:`collections.abc.Coroutine` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. class:: AsyncGenerator(AsyncIterator[YieldType], Generic[YieldType, SendType]) + + Deprecated alias to :class:`collections.abc.AsyncGenerator`. + + An async generator can be annotated by the generic type + ``AsyncGenerator[YieldType, SendType]``. For example:: + + async def echo_round() -> AsyncGenerator[int, float]: + sent = yield 0 + while sent >= 0.0: + rounded = await round(sent) + sent = yield rounded + + Unlike normal generators, async generators cannot return a value, so there + is no ``ReturnType`` type parameter. As with :class:`Generator`, the + ``SendType`` behaves contravariantly. + + If your generator will only yield values, set the ``SendType`` to + ``None``:: + + async def infinite_stream(start: int) -> AsyncGenerator[int, None]: + while True: + yield start + start = await increment(start) + + Alternatively, annotate your generator as having a return type of + either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: + + async def infinite_stream(start: int) -> AsyncIterator[int]: + while True: + yield start + start = await increment(start) + + .. versionadded:: 3.6.1 + + .. deprecated:: 3.9 + :class:`collections.abc.AsyncGenerator` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. class:: AsyncIterable(Generic[T_co]) + + Deprecated alias to :class:`collections.abc.AsyncIterable`. .. versionadded:: 3.5.2 + .. deprecated:: 3.9 + :class:`collections.abc.AsyncIterable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. class:: AsyncIterator(AsyncIterable[T_co]) + + Deprecated alias to :class:`collections.abc.AsyncIterator`. + + .. versionadded:: 3.5.2 + + .. deprecated:: 3.9 + :class:`collections.abc.AsyncIterator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. class:: Awaitable(Generic[T_co]) + + Deprecated alias to :class:`collections.abc.Awaitable`. + + .. versionadded:: 3.5.2 + + .. deprecated:: 3.9 + :class:`collections.abc.Awaitable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. _corresponding-to-other-types-in-collections-abc: + +Aliases to other ABCs in :mod:`collections.abc` +""""""""""""""""""""""""""""""""""""""""""""""" + +.. class:: Iterable(Generic[T_co]) + + Deprecated alias to :class:`collections.abc.Iterable`. + + .. deprecated:: 3.9 + :class:`collections.abc.Iterable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. class:: Iterator(Iterable[T_co]) + + Deprecated alias to :class:`collections.abc.Iterator`. + + .. deprecated:: 3.9 + :class:`collections.abc.Iterator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. data:: Callable + + Deprecated alias to :class:`collections.abc.Callable`. + + See :ref:`annotating-callables` for details on how to use + :class:`collections.abc.Callable` and ``typing.Callable`` in type annotations. + + .. deprecated:: 3.9 + :class:`collections.abc.Callable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + + .. versionchanged:: 3.10 + ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. + See :pep:`612` for more details. + +.. class:: Generator(Iterator[YieldType], Generic[YieldType, SendType, ReturnType]) + + Deprecated alias to :class:`collections.abc.Generator`. + + A generator can be annotated by the generic type + ``Generator[YieldType, SendType, ReturnType]``. For example:: + + def echo_round() -> Generator[int, float, str]: + sent = yield 0 + while sent >= 0: + sent = yield round(sent) + return 'Done' + + Note that unlike many other generics in the typing module, the ``SendType`` + of :class:`Generator` behaves contravariantly, not covariantly or + invariantly. + + If your generator will only yield values, set the ``SendType`` and + ``ReturnType`` to ``None``:: + + def infinite_stream(start: int) -> Generator[int, None, None]: + while True: + yield start + start += 1 + + Alternatively, annotate your generator as having a return type of + either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: + + def infinite_stream(start: int) -> Iterator[int]: + while True: + yield start + start += 1 + + .. deprecated:: 3.9 + :class:`collections.abc.Generator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. class:: Hashable + + Deprecated alias to :class:`collections.abc.Hashable`. + + .. deprecated:: 3.12 + Use :class:`collections.abc.Hashable` directly instead. + +.. class:: Reversible(Iterable[T_co]) + + Deprecated alias to :class:`collections.abc.Reversible`. + + .. deprecated:: 3.9 + :class:`collections.abc.Reversible` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. class:: Sized + + Deprecated alias to :class:`collections.abc.Sized`. + + .. deprecated:: 3.12 + Use :class:`collections.abc.Sized` directly instead. + +.. _context-manager-types: + +Aliases to :mod:`contextlib` ABCs +""""""""""""""""""""""""""""""""" + +.. class:: ContextManager(Generic[T_co]) + + Deprecated alias to :class:`contextlib.AbstractContextManager`. + + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 + + .. deprecated:: 3.9 + :class:`contextlib.AbstractContextManager` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + +.. class:: AsyncContextManager(Generic[T_co]) + + Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. + + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 + + .. deprecated:: 3.9 + :class:`contextlib.AbstractAsyncContextManager` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. + Deprecation Timeline of Major Features ====================================== @@ -2978,19 +3708,38 @@ Certain features in ``typing`` are deprecated and may be removed in a future version of Python. The following table summarizes major deprecations for your convenience. This is subject to change, and not all deprecations are listed. -+----------------------------------+---------------+-------------------+----------------+ -| Feature | Deprecated in | Projected removal | PEP/issue | -+==================================+===============+===================+================+ -| ``typing.io`` and ``typing.re`` | 3.8 | 3.13 | :issue:`38291` | -| submodules | | | | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | -| collections | | | | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | -| ``typing.Sized`` | | | | -+----------------------------------+---------------+-------------------+----------------+ +.. list-table:: + :header-rows: 1 + + * - Feature + - Deprecated in + - Projected removal + - PEP/issue + * - ``typing`` versions of standard collections + - 3.9 + - Undecided (see :ref:`deprecated-aliases` for more information) + - :pep:`585` + * - :class:`typing.ByteString` + - 3.9 + - 3.14 + - :gh:`91896` + * - :data:`typing.Text` + - 3.11 + - Undecided + - :gh:`92332` + * - :class:`typing.Hashable` and :class:`typing.Sized` + - 3.12 + - Undecided + - :gh:`94309` + * - :data:`typing.TypeAlias` + - 3.12 + - Undecided + - :pep:`695` + * - :func:`@typing.no_type_check_decorator ` + - 3.13 + - 3.15 + - :gh:`106309` + * - :data:`typing.AnyStr` + - 3.13 + - 3.18 + - :gh:`105578` diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 3a094f9c64d4a0..7db47d48022a0e 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -17,8 +17,8 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 15.0.0 -`_. +this database is compiled from the `UCD version 15.1.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" @@ -175,6 +175,6 @@ Examples: .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/15.0.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/15.1.0/ucd/NameAliases.txt -.. [#] https://www.unicode.org/Public/15.0.0/ucd/NamedSequences.txt +.. [#] https://www.unicode.org/Public/15.1.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 895b9f9f07671b..f2bdde80bdbd64 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -339,7 +339,7 @@ instantiate the class in those tests. >>> mock.old_method() Traceback (most recent call last): ... - AttributeError: object has no attribute 'old_method' + AttributeError: Mock object has no attribute 'old_method'. Did you mean: 'class_method'? Using a specification also enables a smarter matching of calls made to the mock, regardless of whether some parameters were passed as positional or @@ -360,6 +360,30 @@ of arbitrary attributes as well as the getting of them then you can use *spec_set* instead of *spec*. +Using side_effect to return per file content +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:func:`mock_open` is used to patch :func:`open` method. :attr:`~Mock.side_effect` +can be used to return a new Mock object per call. This can be used to return different +contents per file stored in a dictionary:: + + DEFAULT = "default" + data_dict = {"file1": "data1", + "file2": "data2"} + + def open_side_effect(name): + return mock_open(read_data=data_dict.get(name, DEFAULT))() + + with patch("builtins.open", side_effect=open_side_effect): + with open("file1") as file1: + assert file1.read() == "data1" + + with open("file2") as file2: + assert file2.read() == "data2" + + with open("file3") as file2: + assert file2.read() == "default" + Patch Decorators ---------------- @@ -579,14 +603,14 @@ Partial mocking In some tests I wanted to mock out a call to :meth:`datetime.date.today` to return a known date, but I didn't want to prevent the code under test from creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`date.today` method. +so I couldn't just monkey-patch out the static :meth:`datetime.date.today` method. I found a simple way of doing this that involved effectively wrapping the date class with a mock, but passing through calls to the constructor to the real class (and returning real instances). The :func:`patch decorator ` is used here to -mock out the ``date`` class in the module under test. The :attr:`side_effect` +mock out the ``date`` class in the module under test. The :attr:`~Mock.side_effect` attribute on the mock date class is then set to a lambda function that returns a real date. When the mock date class is called a real date will be constructed and returned by ``side_effect``. :: @@ -766,15 +790,16 @@ mock has a nice API for making assertions about how your mock objects are used. >>> mock.foo_bar.assert_called_with('baz', spam='eggs') If your mock is only being called once you can use the -:meth:`assert_called_once_with` method that also asserts that the -:attr:`call_count` is one. +:meth:`~Mock.assert_called_once_with` method that also asserts that the +:attr:`~Mock.call_count` is one. >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs') >>> mock.foo_bar() >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs') Traceback (most recent call last): ... - AssertionError: Expected to be called once. Called 2 times. + AssertionError: Expected 'foo_bar' to be called once. Called 2 times. + Calls: [call('baz', spam='eggs'), call()]. Both ``assert_called_with`` and ``assert_called_once_with`` make assertions about the *most recent* call. If your mock is going to be called several times, and @@ -835,7 +860,7 @@ One possibility would be for mock to copy the arguments you pass in. This could then cause problems if you do assertions that rely on object identity for equality. -Here's one solution that uses the :attr:`side_effect` +Here's one solution that uses the :attr:`~Mock.side_effect` functionality. If you provide a ``side_effect`` function for a mock then ``side_effect`` will be called with the same args as the mock. This gives us an opportunity to copy the arguments and store them for later assertions. In this @@ -903,8 +928,9 @@ Here's an example implementation: >>> c.assert_called_with(arg) Traceback (most recent call last): ... - AssertionError: Expected call: mock({1}) - Actual call: mock(set()) + AssertionError: expected call not found. + Expected: mock({1}) + Actual: mock(set()) >>> c.foo @@ -971,7 +997,8 @@ We can do this with :class:`MagicMock`, which will behave like a dictionary, and using :data:`~Mock.side_effect` to delegate dictionary access to a real underlying dictionary that is under our control. -When the :meth:`__getitem__` and :meth:`__setitem__` methods of our ``MagicMock`` are called +When the :meth:`~object.__getitem__` and :meth:`~object.__setitem__` methods +of our ``MagicMock`` are called (normal dictionary access) then ``side_effect`` is called with the key (and in the case of ``__setitem__`` the value too). We can also control what is returned. @@ -1267,8 +1294,9 @@ sufficient: >>> mock.assert_called_with(Foo(1, 2)) Traceback (most recent call last): ... - AssertionError: Expected: call(<__main__.Foo object at 0x...>) - Actual call: call(<__main__.Foo object at 0x...>) + AssertionError: expected call not found. + Expected: mock(<__main__.Foo object at 0x...>) + Actual: mock(<__main__.Foo object at 0x...>) A comparison function for our ``Foo`` class might look something like this: diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 6c4d801f69f5a9..d6ac4d09300d9f 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -189,7 +189,7 @@ code if they are used incorrectly: >>> mock_function('wrong arguments') Traceback (most recent call last): ... - TypeError: () takes exactly 3 arguments (1 given) + TypeError: missing a required argument: 'b' :func:`create_autospec` can also be used on classes, where it copies the signature of the ``__init__`` method, and on callable objects where it copies the signature of @@ -205,8 +205,10 @@ The Mock Class import asyncio import inspect import unittest + import threading from unittest.mock import sentinel, DEFAULT, ANY from unittest.mock import patch, call, Mock, MagicMock, PropertyMock, AsyncMock + from unittest.mock import ThreadingMock from unittest.mock import mock_open :class:`Mock` is a flexible mock object intended to replace the use of stubs and @@ -313,6 +315,7 @@ the *new_callable* argument to :func:`patch`. Traceback (most recent call last): ... AssertionError: Expected 'method' to have been called once. Called 2 times. + Calls: [call(), call()]. .. versionadded:: 3.6 @@ -340,7 +343,7 @@ the *new_callable* argument to :func:`patch`. Traceback (most recent call last): ... AssertionError: Expected 'mock' to be called once. Called 2 times. - + Calls: [call('foo', bar='baz'), call('other', bar='values')]. .. method:: assert_any_call(*args, **kwargs) @@ -390,6 +393,7 @@ the *new_callable* argument to :func:`patch`. Traceback (most recent call last): ... AssertionError: Expected 'hello' to not have been called. Called 1 times. + Calls: [call()]. .. versionadded:: 3.5 @@ -814,8 +818,8 @@ This applies to :meth:`~Mock.assert_called_with`, :meth:`~Mock.assert_any_call`. When :ref:`auto-speccing`, it will also apply to method calls on the mock object. - .. versionchanged:: 3.4 - Added signature introspection on specced and autospecced mock objects. +.. versionchanged:: 3.4 + Added signature introspection on specced and autospecced mock objects. .. class:: PropertyMock(*args, **kwargs) @@ -952,7 +956,7 @@ object:: >>> asyncio.run(main()) >>> mock.assert_awaited_once() >>> asyncio.run(main()) - >>> mock.method.assert_awaited_once() + >>> mock.assert_awaited_once() Traceback (most recent call last): ... AssertionError: Expected mock to have been awaited once. Awaited 2 times. @@ -970,7 +974,7 @@ object:: >>> mock.assert_awaited_with('other') Traceback (most recent call last): ... - AssertionError: expected call not found. + AssertionError: expected await not found. Expected: mock('other') Actual: mock('foo', bar='bar') @@ -1099,6 +1103,51 @@ object:: [call('foo'), call('bar')] +.. class:: ThreadingMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, *, timeout=UNSET, **kwargs) + + A version of :class:`MagicMock` for multithreading tests. The + :class:`ThreadingMock` object provides extra methods to wait for a call to + be invoked, rather than assert on it immediately. + + The default timeout is specified by the ``timeout`` argument, or if unset by the + :attr:`ThreadingMock.DEFAULT_TIMEOUT` attribute, which defaults to blocking (``None``). + + You can configure the global default timeout by setting :attr:`ThreadingMock.DEFAULT_TIMEOUT`. + + .. method:: wait_until_called(*, timeout=UNSET) + + Waits until the mock is called. + + If a timeout was passed at the creation of the mock or if a timeout + argument is passed to this function, the function raises an + :exc:`AssertionError` if the call is not performed in time. + + >>> mock = ThreadingMock() + >>> thread = threading.Thread(target=mock) + >>> thread.start() + >>> mock.wait_until_called(timeout=1) + >>> thread.join() + + .. method:: wait_until_any_call_with(*args, **kwargs) + + Waits until the mock is called with the specified arguments. + + If a timeout was passed at the creation of the mock + the function raises an :exc:`AssertionError` if the call is not performed in time. + + >>> mock = ThreadingMock() + >>> thread = threading.Thread(target=mock, args=("arg1", "arg2",), kwargs={"arg": "thing"}) + >>> thread.start() + >>> mock.wait_until_any_call_with("arg1", "arg2", arg="thing") + >>> thread.join() + + .. attribute:: DEFAULT_TIMEOUT + + Global default timeout in seconds to create instances of :class:`ThreadingMock`. + + .. versionadded:: 3.13 + + Calling ~~~~~~~ @@ -1388,9 +1437,9 @@ patch .. note:: - .. versionchanged:: 3.5 - If you are patching builtins in a module then you don't - need to pass ``create=True``, it will be added by default. + .. versionchanged:: 3.5 + If you are patching builtins in a module then you don't + need to pass ``create=True``, it will be added by default. Patch can be used as a :class:`TestCase` class decorator. It works by decorating each test method in the class. This reduces the boilerplate @@ -1658,7 +1707,7 @@ Keywords can be used in the :func:`patch.dict` call to set values in the diction :func:`patch.dict` can be used with dictionary like objects that aren't actually dictionaries. At the very minimum they must support item getting, setting, deleting and either iteration or membership test. This corresponds to the -magic methods :meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either +magic methods :meth:`~object.__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either :meth:`__iter__` or :meth:`__contains__`. >>> class Container: @@ -2438,7 +2487,7 @@ behaviour you can switch it off by setting the module level switch Alternatively you can just use ``vars(my_mock)`` (instance members) and ``dir(type(my_mock))`` (type members) to bypass the filtering irrespective of -:data:`mock.FILTER_DIR`. +:const:`mock.FILTER_DIR`. mock_open @@ -2482,8 +2531,8 @@ are closed properly and is becoming common:: f.write('something') The issue is that even if you mock out the call to :func:`open` it is the -*returned object* that is used as a context manager (and has :meth:`__enter__` and -:meth:`__exit__` called). +*returned object* that is used as a context manager (and has :meth:`~object.__enter__` and +:meth:`~object.__exit__` called). Mocking context managers with a :class:`MagicMock` is common enough and fiddly enough that a helper function is useful. :: diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index b26e6c0e6bc024..02b72cb9f6b8aa 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -206,13 +206,13 @@ Command-line options .. program:: unittest -.. cmdoption:: -b, --buffer +.. option:: -b, --buffer The standard output and standard error streams are buffered during the test run. Output during a passing test is discarded. Output is echoed normally on test fail or error and is added to the failure messages. -.. cmdoption:: -c, --catch +.. option:: -c, --catch :kbd:`Control-C` during the test run waits for the current test to end and then reports all the results so far. A second :kbd:`Control-C` raises the normal @@ -220,11 +220,11 @@ Command-line options See `Signal Handling`_ for the functions that provide this functionality. -.. cmdoption:: -f, --failfast +.. option:: -f, --failfast Stop the test run on the first error or failure. -.. cmdoption:: -k +.. option:: -k Only run test methods and classes that match the pattern or substring. This option may be used multiple times, in which case all test cases that @@ -240,11 +240,11 @@ Command-line options For example, ``-k foo`` matches ``foo_tests.SomeTest.test_something``, ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. -.. cmdoption:: --locals +.. option:: --locals Show local variables in tracebacks. -.. cmdoption:: --durations N +.. option:: --durations N Show the N slowest test cases (N=0 for all). @@ -292,19 +292,19 @@ The ``discover`` sub-command has the following options: .. program:: unittest discover -.. cmdoption:: -v, --verbose +.. option:: -v, --verbose Verbose output -.. cmdoption:: -s, --start-directory directory +.. option:: -s, --start-directory directory Directory to start discovery (``.`` default) -.. cmdoption:: -p, --pattern pattern +.. option:: -p, --pattern pattern Pattern to match test files (``test*.py`` default) -.. cmdoption:: -t, --top-level-directory directory +.. option:: -t, --top-level-directory directory Top level directory of project (defaults to start directory) @@ -1134,7 +1134,7 @@ Test cases If given, *level* should be either a numeric logging level or its string equivalent (for example either ``"ERROR"`` or - :attr:`logging.ERROR`). The default is :attr:`logging.INFO`. + :const:`logging.ERROR`). The default is :const:`logging.INFO`. The test passes if at least one message emitted inside the ``with`` block matches the *logger* and *level* conditions, otherwise it fails. @@ -1175,7 +1175,7 @@ Test cases If given, *level* should be either a numeric logging level or its string equivalent (for example either ``"ERROR"`` or - :attr:`logging.ERROR`). The default is :attr:`logging.INFO`. + :const:`logging.ERROR`). The default is :const:`logging.INFO`. Unlike :meth:`assertLogs`, nothing will be returned by the context manager. @@ -1571,6 +1571,14 @@ Test cases .. versionadded:: 3.8 + .. attribute:: loop_factory + + The *loop_factory* passed to :class:`asyncio.Runner`. Override + in subclasses with :class:`asyncio.EventLoop` to avoid using the + asyncio policy system. + + .. versionadded:: 3.13 + .. coroutinemethod:: asyncSetUp() Method called to prepare the test fixture. This is called after :meth:`setUp`. @@ -2017,7 +2025,7 @@ Loading and running tests .. attribute:: collectedDurations - A list containing 2-tuples of :class:`TestCase` instances and floats + A list containing 2-tuples of test case names and floats representing the elapsed time of each test which was run. .. versionadded:: 3.12 @@ -2318,6 +2326,8 @@ Loading and running tests test names. +.. _load_tests-protocol: + load_tests Protocol ################### diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index 3adbdd26132273..facb11f42a40c5 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -27,8 +27,8 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: exception instance. .. versionchanged:: 3.3 - :exc:`URLError` has been made a subclass of :exc:`OSError` instead - of :exc:`IOError`. + :exc:`URLError` used to be a subtype of :exc:`IOError`, which is now an + alias of :exc:`OSError`. .. exception:: HTTPError(url, code, msg, hdrs, fp) @@ -72,6 +72,8 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: This exception is raised when the :func:`~urllib.request.urlretrieve` function detects that the amount of the downloaded data is less than the expected amount (given by - the *Content-Length* header). The :attr:`content` attribute stores the - downloaded (and supposedly truncated) data. + the *Content-Length* header). + .. attribute:: content + + The downloaded (and supposedly truncated) data. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 96b396510794b4..53e5f0395715d7 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -23,9 +23,9 @@ to an absolute URL given a "base URL." The module has been designed to match the internet RFC on Relative Uniform Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, ``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, -``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtspu``, ``sftp``, -``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, -``wais``, ``ws``, ``wss``. +``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtsps``, ``rtspu``, +``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, +``telnet``, ``wais``, ``ws``, ``wss``. The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in @@ -159,6 +159,10 @@ or on combining URL components into a URL string. ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') + .. warning:: + + :func:`urlparse` does not perform validation. See :ref:`URL parsing + security ` for details. .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -324,8 +328,14 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. - Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline - ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. + Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 + control and space characters are stripped from the URL. ``\n``, + ``\r`` and tab ``\t`` characters are removed from the URL at any position. + + .. warning:: + + :func:`urlsplit` does not perform validation. See :ref:`URL parsing + security ` for details. .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of @@ -338,6 +348,9 @@ or on combining URL components into a URL string. .. versionchanged:: 3.10 ASCII newline and tab characters are stripped from the URL. + .. versionchanged:: 3.12 + Leading WHATWG C0 control and space characters are stripped from the URL. + .. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser .. function:: urlunsplit(parts) @@ -414,6 +427,35 @@ or on combining URL components into a URL string. or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned without changes. +.. _url-parsing-security: + +URL parsing security +-------------------- + +The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of +inputs. They may not raise errors on inputs that other applications consider +invalid. They may also succeed on some inputs that might not be considered +URLs elsewhere. Their purpose is for practical functionality rather than +purity. + +Instead of raising an exception on unusual input, they may instead return some +component parts as empty strings. Or components may contain more than perhaps +they should. + +We recommend that users of these APIs where the values may be used anywhere +with security implications code defensively. Do some verification within your +code before trusting a returned component part. Does that ``scheme`` make +sense? Is that a sensible ``path``? Is there anything strange about that +``hostname``? etc. + +What constitutes a URL is not universally well defined. Different applications +have different needs and desired constraints. For instance the living `WHATWG +spec`_ describes what user facing web clients such as a web browser require. +While :rfc:`3986` is more general. These functions incorporate some aspects of +both, but cannot be claimed compliant with either. The APIs and existing user +code with expectations on specific behaviors predate both standards leading us +to be very cautious about making API behavior changes. + .. _parsing-ascii-encoded-bytes: Parsing ASCII Encoded Bytes @@ -556,7 +598,7 @@ task isn't already covered by the URL parsing functions above. .. function:: quote(string, safe='/', encoding=None, errors=None) - Replace special characters in *string* using the ``%xx`` escape. Letters, + Replace special characters in *string* using the :samp:`%{xx}` escape. Letters, digits, and the characters ``'_.-~'`` are never quoted. By default, this function is intended for quoting the path section of a URL. The optional *safe* parameter specifies additional ASCII characters that should not be @@ -603,7 +645,7 @@ task isn't already covered by the URL parsing functions above. .. function:: unquote(string, encoding='utf-8', errors='replace') - Replace ``%xx`` escapes with their single-character equivalent. + Replace :samp:`%{xx}` escapes with their single-character equivalent. The optional *encoding* and *errors* parameters specify how to decode percent-encoded sequences into Unicode characters, as accepted by the :meth:`bytes.decode` method. @@ -634,7 +676,7 @@ task isn't already covered by the URL parsing functions above. .. function:: unquote_to_bytes(string) - Replace ``%xx`` escapes with their single-octet equivalent, and return a + Replace :samp:`%{xx}` escapes with their single-octet equivalent, and return a :class:`bytes` object. *string* may be either a :class:`str` or a :class:`bytes` object. diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 1b05458280d896..bf3af1bef0714c 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -26,7 +26,7 @@ authentication, redirections, cookies and more. The :mod:`urllib.request` module defines the following functions: -.. function:: urlopen(url, data=None[, timeout], *, cafile=None, capath=None, cadefault=False, context=None) +.. function:: urlopen(url, data=None[, timeout], *, context=None) Open *url*, which can be either a string containing a valid, properly encoded URL, or a :class:`Request` object. @@ -47,14 +47,6 @@ The :mod:`urllib.request` module defines the following functions: describing the various SSL options. See :class:`~http.client.HTTPSConnection` for more details. - The optional *cafile* and *capath* parameters specify a set of trusted - CA certificates for HTTPS requests. *cafile* should point to a single - file containing a bundle of CA certificates, whereas *capath* should - point to a directory of hashed certificate files. More information can - be found in :meth:`ssl.SSLContext.load_verify_locations`. - - The *cadefault* parameter is ignored. - This function always returns an object which can work as a :term:`context manager` and has the properties *url*, *headers*, and *status*. See :class:`urllib.response.addinfourl` for more detail on these properties. @@ -99,7 +91,7 @@ The :mod:`urllib.request` module defines the following functions: .. versionchanged:: 3.2 HTTPS virtual hosts are now supported if possible (that is, if - :data:`ssl.HAS_SNI` is true). + :const:`ssl.HAS_SNI` is true). .. versionadded:: 3.2 *data* can be an iterable object. @@ -115,12 +107,9 @@ The :mod:`urllib.request` module defines the following functions: ``http/1.1`` when no *context* is given. Custom *context* should set ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. - .. deprecated:: 3.6 - - *cafile*, *capath* and *cadefault* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. + .. versionchanged:: 3.13 + Remove *cafile*, *capath* and *cadefault* parameters: use the *context* + parameter instead. .. function:: install_opener(opener) @@ -315,10 +304,10 @@ The following classes are provided: list of hostname suffixes, optionally with ``:port`` appended, for example ``cern.ch,ncsa.uiuc.edu,some.host:8080``. - .. note:: + .. note:: - ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; - see the documentation on :func:`~urllib.request.getproxies`. + ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; + see the documentation on :func:`~urllib.request.getproxies`. .. class:: HTTPPasswordMgr() @@ -1536,9 +1525,9 @@ some point in the future. :mod:`urllib.request` Restrictions ---------------------------------- - .. index:: - pair: HTTP; protocol - pair: FTP; protocol +.. index:: + pair: HTTP; protocol + pair: FTP; protocol * Currently, only the following protocols are supported: HTTP (versions 0.9 and 1.0), FTP, local files, and data URLs. diff --git a/Doc/library/uu.rst b/Doc/library/uu.rst deleted file mode 100644 index 83c4aec47bbefc..00000000000000 --- a/Doc/library/uu.rst +++ /dev/null @@ -1,72 +0,0 @@ -:mod:`uu` --- Encode and decode uuencode files -============================================== - -.. module:: uu - :synopsis: Encode and decode files in uuencode format. - :deprecated: - -.. moduleauthor:: Lance Ellinghouse - -**Source code:** :source:`Lib/uu.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`uu` module is deprecated - (see :pep:`PEP 594 <594#uu-and-the-uu-encoding>` for details). - :mod:`base64` is a modern alternative. - --------------- - -This module encodes and decodes files in uuencode format, allowing arbitrary -binary data to be transferred over ASCII-only connections. Wherever a file -argument is expected, the methods accept a file-like object. For backwards -compatibility, a string containing a pathname is also accepted, and the -corresponding file will be opened for reading and writing; the pathname ``'-'`` -is understood to mean the standard input or output. However, this interface is -deprecated; it's better for the caller to open the file itself, and be sure -that, when required, the mode is ``'rb'`` or ``'wb'`` on Windows. - -.. index:: - single: Jansen, Jack - single: Ellinghouse, Lance - -This code was contributed by Lance Ellinghouse, and modified by Jack Jansen. - -The :mod:`uu` module defines the following functions: - - -.. function:: encode(in_file, out_file, name=None, mode=None, *, backtick=False) - - Uuencode file *in_file* into file *out_file*. The uuencoded file will have - the header specifying *name* and *mode* as the defaults for the results of - decoding the file. The default defaults are taken from *in_file*, or ``'-'`` - and ``0o666`` respectively. If *backtick* is true, zeros are represented by - ``'`'`` instead of spaces. - - .. versionchanged:: 3.7 - Added the *backtick* parameter. - - -.. function:: decode(in_file, out_file=None, mode=None, quiet=False) - - This call decodes uuencoded file *in_file* placing the result on file - *out_file*. If *out_file* is a pathname, *mode* is used to set the permission - bits if the file must be created. Defaults for *out_file* and *mode* are taken - from the uuencode header. However, if the file specified in the header already - exists, a :exc:`uu.Error` is raised. - - :func:`decode` may print a warning to standard error if the input was produced - by an incorrect uuencoder and Python could recover from that error. Setting - *quiet* to a true value silences this warning. - - -.. exception:: Error() - - Subclass of :exc:`Exception`, this can be raised by :func:`uu.decode` under - various situations, such as described above, but also including a badly - formatted header, or truncated input file. - - -.. seealso:: - - Module :mod:`binascii` - Support module containing ASCII-to-binary and binary-to-ASCII conversions. diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index 94b9a432372195..e2d231da38fd9a 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -22,7 +22,7 @@ random UUID. Depending on support from the underlying platform, :func:`uuid1` may or may not return a "safe" UUID. A safe UUID is one which is generated using synchronization methods that ensure no two processes can obtain the same -UUID. All instances of :class:`UUID` have an :attr:`is_safe` attribute +UUID. All instances of :class:`UUID` have an :attr:`~UUID.is_safe` attribute which relays any information about the UUID's safety, using this enumeration: .. class:: SafeUUID @@ -95,25 +95,34 @@ which relays any information about the UUID's safety, using this enumeration: A tuple of the six integer fields of the UUID, which are also available as six individual attributes and two derived attributes: - +------------------------------+-------------------------------+ - | Field | Meaning | - +==============================+===============================+ - | :attr:`time_low` | the first 32 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time_mid` | the next 16 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time_hi_version` | the next 16 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`clock_seq_hi_variant` | the next 8 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`clock_seq_low` | the next 8 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`node` | the last 48 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time` | the 60-bit timestamp | - +------------------------------+-------------------------------+ - | :attr:`clock_seq` | the 14-bit sequence number | - +------------------------------+-------------------------------+ +.. list-table:: + + * - Field + - Meaning + + * - .. attribute:: UUID.time_low + - The first 32 bits of the UUID. + + * - .. attribute:: UUID.time_mid + - The next 16 bits of the UUID. + + * - .. attribute:: UUID.time_hi_version + - The next 16 bits of the UUID. + + * - .. attribute:: UUID.clock_seq_hi_variant + - The next 8 bits of the UUID. + + * - .. attribute:: UUID.clock_seq_low + - The next 8 bits of the UUID. + + * - .. attribute:: UUID.node + - The last 48 bits of the UUID. + + * - .. attribute:: UUID.time + - The 60-bit timestamp. + + * - .. attribute:: UUID.clock_seq + - The 14-bit sequence number. .. attribute:: UUID.hex @@ -233,7 +242,7 @@ The :mod:`uuid` module defines the following namespace identifiers for use with text output format. The :mod:`uuid` module defines the following constants for the possible values -of the :attr:`variant` attribute: +of the :attr:`~UUID.variant` attribute: .. data:: RESERVED_NCS @@ -280,25 +289,25 @@ The following options are accepted: .. program:: uuid -.. cmdoption:: -h, --help +.. option:: -h, --help Show the help message and exit. -.. cmdoption:: -u - --uuid +.. option:: -u + --uuid Specify the function name to use to generate the uuid. By default :func:`uuid4` is used. -.. cmdoption:: -n - --namespace +.. option:: -n + --namespace The namespace is a ``UUID``, or ``@ns`` where ``ns`` is a well-known predefined UUID addressed by namespace name. Such as ``@dns``, ``@url``, ``@oid``, and ``@x500``. Only required for :func:`uuid3` / :func:`uuid5` functions. -.. cmdoption:: -N - --name +.. option:: -N + --name The name used as part of generating the uuid. Only required for :func:`uuid3` / :func:`uuid5` functions. diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 9e5672545dea35..da8942c554dea1 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -30,6 +30,25 @@ When used from within a virtual environment, common installation tools such as `pip`_ will install Python packages into a virtual environment without needing to be told to do so explicitly. +A virtual environment is (amongst other things): + +* Used to contain a specific Python interpreter and software libraries and + binaries which are needed to support a project (library or application). These + are by default isolated from software in other virtual environments and Python + interpreters and libraries installed in the operating system. + +* Contained in a directory, conventionally either named ``venv`` or ``.venv`` in + the project directory, or under a container directory for lots of virtual + environments, such as ``~/.virtualenvs``. + +* Not checked into source control systems such as Git. + +* Considered as disposable -- it should be simple to delete and recreate it from + scratch. You don't place any project code in the environment + +* Not considered as movable or copyable -- you just recreate the same + environment in the target location. + See :pep:`405` for more background on Python virtual environments. .. seealso:: @@ -60,7 +79,7 @@ running from a virtual environment. A virtual environment may be "activated" using a script in its binary directory (``bin`` on POSIX; ``Scripts`` on Windows). -This will prepend that directory to your :envvar:`!PATH`, so that running +This will prepend that directory to your :envvar:`PATH`, so that running :program:`python` will invoke the environment's Python interpreter and you can run installed scripts without having to use their full path. The invocation of the activation script is platform-specific @@ -100,10 +119,10 @@ In order to achieve this, scripts installed into virtual environments have a "shebang" line which points to the environment's Python interpreter, i.e. :samp:`#!/{}/bin/python`. This means that the script will run with that interpreter regardless of the -value of :envvar:`!PATH`. On Windows, "shebang" line processing is supported if +value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if you have the :ref:`launcher` installed. Thus, double-clicking an installed script in a Windows Explorer window should run it with the correct interpreter -without the environment needing to be activated or on the :envvar:`!PATH`. +without the environment needing to be activated or on the :envvar:`PATH`. When a virtual environment has been activated, the :envvar:`!VIRTUAL_ENV` environment variable is set to the path of the environment. @@ -143,7 +162,8 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False) + prompt=None, upgrade_deps=False, \ + *, scm_ignore_files=frozenset()) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -172,6 +192,12 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + * ``scm_ignore_files`` -- Create ignore files based for the specified source + control managers (SCM) in the iterable. Support is defined by having a + method named ``create_{scm}_ignore_file``. The only value supported by + default is ``"git"`` via :meth:`create_git_ignore_file`. + + .. versionchanged:: 3.4 Added the ``with_pip`` parameter @@ -181,6 +207,9 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionadded:: 3.9 Added the ``upgrade_deps`` parameter + .. versionadded:: 3.13 + Added the ``scm_ignore_files`` parameter + Creators of third-party virtual environment tools will be free to use the provided :class:`EnvBuilder` class as a base class. @@ -339,11 +368,18 @@ creation according to their needs, the :class:`EnvBuilder` class. The directories are allowed to exist (for when an existing environment is being upgraded). + .. method:: create_git_ignore_file(context) + + Creates a ``.gitignore`` file within the virtual environment that causes + the entire directory to be ignored by the ``git`` source control manager. + + .. versionadded:: 3.13 + There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False) + upgrade_deps=False, *, scm_ignore_files=frozenset()) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. @@ -359,6 +395,9 @@ There is also a module-level convenience function: .. versionchanged:: 3.9 Added the ``upgrade_deps`` parameter + .. versionchanged:: 3.13 + Added the ``scm_ignore_files`` parameter + An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 884de08eab1b16..a9c469707e8227 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -522,6 +522,56 @@ Available Functions and calls to :func:`simplefilter`. +.. decorator:: deprecated(msg, *, category=DeprecationWarning, stacklevel=1) + + Decorator to indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, + deprecation warnings may be emitted at runtime when the object is used. + :term:`static type checkers ` + will also generate a diagnostic on usage of the deprecated object. + + Usage:: + + from warnings import deprecated + from typing import overload + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the :func:`@overload ` decorator + for the attribute to exist on the overload as returned by + :func:`typing.get_overloads`. + + .. versionadded:: 3.13 + See :pep:`702`. + + Available Context Managers -------------------------- diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 04a28d97d619eb..55b029bc742b24 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -11,8 +11,9 @@ -------------- -The :mod:`wave` module provides a convenient interface to the WAV sound format. -Only PCM encoded wave files are supported. +The :mod:`wave` module provides a convenient interface to the Waveform Audio +"WAVE" (or "WAV") file format. Only uncompressed PCM encoded wave files are +supported. .. versionchanged:: 3.12 @@ -41,13 +42,12 @@ The :mod:`wave` module defines the following function and exception: value for *mode*. If you pass in a file-like object, the wave object will not close it when its - :meth:`close` method is called; it is the caller's responsibility to close + ``close()`` method is called; it is the caller's responsibility to close the file object. The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`!with` block completes, the :meth:`Wave_read.close() - ` or :meth:`Wave_write.close() - ` method is called. + the :keyword:`!with` block completes, the :meth:`Wave_read.close()` or + :meth:`Wave_write.close()` method is called. .. versionchanged:: 3.4 Added support for unseekable files. @@ -63,87 +63,99 @@ The :mod:`wave` module defines the following function and exception: Wave_read Objects ----------------- -Wave_read objects, as returned by :func:`.open`, have the following methods: +.. class:: Wave_read + Read a WAV file. -.. method:: Wave_read.close() + Wave_read objects, as returned by :func:`.open`, have the following methods: - Close the stream if it was opened by :mod:`wave`, and make the instance - unusable. This is called automatically on object collection. + .. method:: close() -.. method:: Wave_read.getnchannels() + Close the stream if it was opened by :mod:`wave`, and make the instance + unusable. This is called automatically on object collection. - Returns number of audio channels (``1`` for mono, ``2`` for stereo). + .. method:: getnchannels() -.. method:: Wave_read.getsampwidth() + Returns number of audio channels (``1`` for mono, ``2`` for stereo). - Returns sample width in bytes. + .. method:: getsampwidth() -.. method:: Wave_read.getframerate() + Returns sample width in bytes. - Returns sampling frequency. + .. method:: getframerate() -.. method:: Wave_read.getnframes() + Returns sampling frequency. - Returns number of audio frames. + .. method:: getnframes() -.. method:: Wave_read.getcomptype() + Returns number of audio frames. - Returns compression type (``'NONE'`` is the only supported type). + .. method:: getcomptype() -.. method:: Wave_read.getcompname() + Returns compression type (``'NONE'`` is the only supported type). - Human-readable version of :meth:`getcomptype`. Usually ``'not compressed'`` - parallels ``'NONE'``. + .. method:: getcompname() -.. method:: Wave_read.getparams() + Human-readable version of :meth:`getcomptype`. Usually ``'not compressed'`` + parallels ``'NONE'``. - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. + .. method:: getparams() -.. method:: Wave_read.readframes(n) + Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, + framerate, nframes, comptype, compname)``, equivalent to output of the + ``get*()`` methods. - Reads and returns at most *n* frames of audio, as a :class:`bytes` object. + .. method:: readframes(n) -.. method:: Wave_read.rewind() + Reads and returns at most *n* frames of audio, as a :class:`bytes` object. - Rewind the file pointer to the beginning of the audio stream. -The following two methods are defined for compatibility with the :mod:`aifc` -module, and don't do anything interesting. + .. method:: rewind() + Rewind the file pointer to the beginning of the audio stream. -.. method:: Wave_read.getmarkers() + The following two methods are defined for compatibility with the old :mod:`!aifc` + module, and don't do anything interesting. - Returns ``None``. + .. method:: getmarkers() -.. method:: Wave_read.getmark(id) + Returns ``None``. - Raise an error. + .. deprecated-removed:: 3.13 3.15 + The method only existed for compatibility with the :mod:`!aifc` module + which has been removed in Python 3.13. -The following two methods define a term "position" which is compatible between -them, and is otherwise implementation dependent. + .. method:: getmark(id) -.. method:: Wave_read.setpos(pos) + Raise an error. - Set the file pointer to the specified position. + .. deprecated-removed:: 3.13 3.15 + The method only existed for compatibility with the :mod:`!aifc` module + which has been removed in Python 3.13. + The following two methods define a term "position" which is compatible between + them, and is otherwise implementation dependent. -.. method:: Wave_read.tell() - Return current file pointer position. + .. method:: setpos(pos) + + Set the file pointer to the specified position. + + + .. method:: tell() + + Return current file pointer position. .. _wave-write-objects: @@ -151,97 +163,100 @@ them, and is otherwise implementation dependent. Wave_write Objects ------------------ -For seekable output streams, the ``wave`` header will automatically be updated -to reflect the number of frames actually written. For unseekable streams, the -*nframes* value must be accurate when the first frame data is written. An -accurate *nframes* value can be achieved either by calling -:meth:`~Wave_write.setnframes` or :meth:`~Wave_write.setparams` with the number -of frames that will be written before :meth:`~Wave_write.close` is called and -then using :meth:`~Wave_write.writeframesraw` to write the frame data, or by -calling :meth:`~Wave_write.writeframes` with all of the frame data to be -written. In the latter case :meth:`~Wave_write.writeframes` will calculate -the number of frames in the data and set *nframes* accordingly before writing -the frame data. +.. class:: Wave_write -Wave_write objects, as returned by :func:`.open`, have the following methods: + Write a WAV file. -.. versionchanged:: 3.4 - Added support for unseekable files. + Wave_write objects, as returned by :func:`.open`. + For seekable output streams, the ``wave`` header will automatically be updated + to reflect the number of frames actually written. For unseekable streams, the + *nframes* value must be accurate when the first frame data is written. An + accurate *nframes* value can be achieved either by calling + :meth:`setnframes` or :meth:`setparams` with the number + of frames that will be written before :meth:`close` is called and + then using :meth:`writeframesraw` to write the frame data, or by + calling :meth:`writeframes` with all of the frame data to be + written. In the latter case :meth:`writeframes` will calculate + the number of frames in the data and set *nframes* accordingly before writing + the frame data. -.. method:: Wave_write.close() + .. versionchanged:: 3.4 + Added support for unseekable files. - Make sure *nframes* is correct, and close the file if it was opened by - :mod:`wave`. This method is called upon object collection. It will raise - an exception if the output stream is not seekable and *nframes* does not - match the number of frames actually written. + Wave_write objects have the following methods: + .. method:: close() -.. method:: Wave_write.setnchannels(n) + Make sure *nframes* is correct, and close the file if it was opened by + :mod:`wave`. This method is called upon object collection. It will raise + an exception if the output stream is not seekable and *nframes* does not + match the number of frames actually written. - Set the number of channels. + .. method:: setnchannels(n) -.. method:: Wave_write.setsampwidth(n) + Set the number of channels. - Set the sample width to *n* bytes. + .. method:: setsampwidth(n) -.. method:: Wave_write.setframerate(n) + Set the sample width to *n* bytes. - Set the frame rate to *n*. - .. versionchanged:: 3.2 - A non-integral input to this method is rounded to the nearest - integer. + .. method:: setframerate(n) + Set the frame rate to *n*. -.. method:: Wave_write.setnframes(n) + .. versionchanged:: 3.2 + A non-integral input to this method is rounded to the nearest + integer. - Set the number of frames to *n*. This will be changed later if the number - of frames actually written is different (this update attempt will - raise an error if the output stream is not seekable). + .. method:: setnframes(n) -.. method:: Wave_write.setcomptype(type, name) + Set the number of frames to *n*. This will be changed later if the number + of frames actually written is different (this update attempt will + raise an error if the output stream is not seekable). - Set the compression type and description. At the moment, only compression type - ``NONE`` is supported, meaning no compression. + .. method:: setcomptype(type, name) -.. method:: Wave_write.setparams(tuple) + Set the compression type and description. At the moment, only compression type + ``NONE`` is supported, meaning no compression. - The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, with values valid for the :meth:`set\*` methods. Sets all - parameters. + .. method:: setparams(tuple) -.. method:: Wave_write.tell() + The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, + compname)``, with values valid for the ``set*()`` methods. Sets all + parameters. - Return current position in the file, with the same disclaimer for the - :meth:`Wave_read.tell` and :meth:`Wave_read.setpos` methods. + .. method:: tell() -.. method:: Wave_write.writeframesraw(data) + Return current position in the file, with the same disclaimer for the + :meth:`Wave_read.tell` and :meth:`Wave_read.setpos` methods. - Write audio frames, without correcting *nframes*. - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. + .. method:: writeframesraw(data) + Write audio frames, without correcting *nframes*. -.. method:: Wave_write.writeframes(data) + .. versionchanged:: 3.4 + Any :term:`bytes-like object` is now accepted. - Write audio frames and make sure *nframes* is correct. It will raise an - error if the output stream is not seekable and the total number of frames - that have been written after *data* has been written does not match the - previously set value for *nframes*. - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. + .. method:: writeframes(data) + Write audio frames and make sure *nframes* is correct. It will raise an + error if the output stream is not seekable and the total number of frames + that have been written after *data* has been written does not match the + previously set value for *nframes*. -Note that it is invalid to set any parameters after calling :meth:`writeframes` -or :meth:`writeframesraw`, and any attempt to do so will raise -:exc:`wave.Error`. + .. versionchanged:: 3.4 + Any :term:`bytes-like object` is now accepted. + Note that it is invalid to set any parameters after calling :meth:`writeframes` + or :meth:`writeframesraw`, and any attempt to do so will raise + :exc:`wave.Error`. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 1406b663c6a8e2..d6e062df945c64 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -111,7 +111,7 @@ See :ref:`__slots__ documentation ` for details. Exceptions raised by the callback will be noted on the standard error output, but cannot be propagated; they are handled in exactly the same way as exceptions - raised from an object's :meth:`__del__` method. + raised from an object's :meth:`~object.__del__` method. Weak references are :term:`hashable` if the *object* is hashable. They will maintain their hash value even after the *object* was deleted. If @@ -221,8 +221,7 @@ than needed. Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. :class:`WeakValueDictionary` objects have an additional method that has the -same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` -objects. +same issues as the :meth:`WeakKeyDictionary.keyrefs` method. .. method:: WeakValueDictionary.valuerefs() @@ -281,7 +280,7 @@ objects. Exceptions raised by finalizer callbacks during garbage collection will be shown on the standard error output, but cannot be propagated. They are handled in the same way as exceptions raised - from an object's :meth:`__del__` method or a weak reference's + from an object's :meth:`~object.__del__` method or a weak reference's callback. When the program exits, each remaining live finalizer is called @@ -523,18 +522,18 @@ is still alive. For instance obj dead or exiting -Comparing finalizers with :meth:`__del__` methods -------------------------------------------------- +Comparing finalizers with :meth:`~object.__del__` methods +--------------------------------------------------------- Suppose we want to create a class whose instances represent temporary directories. The directories should be deleted with their contents when the first of the following events occurs: * the object is garbage collected, -* the object's :meth:`remove` method is called, or +* the object's :meth:`!remove` method is called, or * the program exits. -We might try to implement the class using a :meth:`__del__` method as +We might try to implement the class using a :meth:`~object.__del__` method as follows:: class TempDir: @@ -553,12 +552,12 @@ follows:: def __del__(self): self.remove() -Starting with Python 3.4, :meth:`__del__` methods no longer prevent +Starting with Python 3.4, :meth:`~object.__del__` methods no longer prevent reference cycles from being garbage collected, and module globals are no longer forced to :const:`None` during :term:`interpreter shutdown`. So this code should work without any issues on CPython. -However, handling of :meth:`__del__` methods is notoriously implementation +However, handling of :meth:`~object.__del__` methods is notoriously implementation specific, since it depends on internal details of the interpreter's garbage collector implementation. diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 61db8042093627..4667b81e38ada2 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -111,41 +111,41 @@ for the controller classes, all defined in this module. +------------------------+-----------------------------------------+-------+ | Type Name | Class Name | Notes | +========================+=========================================+=======+ -| ``'mozilla'`` | :class:`Mozilla('mozilla')` | | +| ``'mozilla'`` | ``Mozilla('mozilla')`` | | +------------------------+-----------------------------------------+-------+ -| ``'firefox'`` | :class:`Mozilla('mozilla')` | | +| ``'firefox'`` | ``Mozilla('mozilla')`` | | +------------------------+-----------------------------------------+-------+ -| ``'epiphany'`` | :class:`Epiphany('epiphany')` | | +| ``'epiphany'`` | ``Epiphany('epiphany')`` | | +------------------------+-----------------------------------------+-------+ -| ``'kfmclient'`` | :class:`Konqueror()` | \(1) | +| ``'kfmclient'`` | ``Konqueror()`` | \(1) | +------------------------+-----------------------------------------+-------+ -| ``'konqueror'`` | :class:`Konqueror()` | \(1) | +| ``'konqueror'`` | ``Konqueror()`` | \(1) | +------------------------+-----------------------------------------+-------+ -| ``'kfm'`` | :class:`Konqueror()` | \(1) | +| ``'kfm'`` | ``Konqueror()`` | \(1) | +------------------------+-----------------------------------------+-------+ -| ``'opera'`` | :class:`Opera()` | | +| ``'opera'`` | ``Opera()`` | | +------------------------+-----------------------------------------+-------+ -| ``'links'`` | :class:`GenericBrowser('links')` | | +| ``'links'`` | ``GenericBrowser('links')`` | | +------------------------+-----------------------------------------+-------+ -| ``'elinks'`` | :class:`Elinks('elinks')` | | +| ``'elinks'`` | ``Elinks('elinks')`` | | +------------------------+-----------------------------------------+-------+ -| ``'lynx'`` | :class:`GenericBrowser('lynx')` | | +| ``'lynx'`` | ``GenericBrowser('lynx')`` | | +------------------------+-----------------------------------------+-------+ -| ``'w3m'`` | :class:`GenericBrowser('w3m')` | | +| ``'w3m'`` | ``GenericBrowser('w3m')`` | | +------------------------+-----------------------------------------+-------+ -| ``'windows-default'`` | :class:`WindowsDefault` | \(2) | +| ``'windows-default'`` | ``WindowsDefault`` | \(2) | +------------------------+-----------------------------------------+-------+ -| ``'macosx'`` | :class:`MacOSXOSAScript('default')` | \(3) | +| ``'macosx'`` | ``MacOSXOSAScript('default')`` | \(3) | +------------------------+-----------------------------------------+-------+ -| ``'safari'`` | :class:`MacOSXOSAScript('safari')` | \(3) | +| ``'safari'`` | ``MacOSXOSAScript('safari')`` | \(3) | +------------------------+-----------------------------------------+-------+ -| ``'google-chrome'`` | :class:`Chrome('google-chrome')` | | +| ``'google-chrome'`` | ``Chrome('google-chrome')`` | | +------------------------+-----------------------------------------+-------+ -| ``'chrome'`` | :class:`Chrome('chrome')` | | +| ``'chrome'`` | ``Chrome('chrome')`` | | +------------------------+-----------------------------------------+-------+ -| ``'chromium'`` | :class:`Chromium('chromium')` | | +| ``'chromium'`` | ``Chromium('chromium')`` | | +------------------------+-----------------------------------------+-------+ -| ``'chromium-browser'`` | :class:`Chromium('chromium-browser')` | | +| ``'chromium-browser'`` | ``Chromium('chromium-browser')`` | | +------------------------+-----------------------------------------+-------+ Notes: @@ -153,7 +153,7 @@ Notes: (1) "Konqueror" is the file manager for the KDE desktop environment for Unix, and only makes sense to use if KDE is running. Some way of reliably detecting KDE - would be nice; the :envvar:`KDEDIR` variable is not sufficient. Note also that + would be nice; the :envvar:`!KDEDIR` variable is not sufficient. Note also that the name "kfm" is used even when using the :program:`konqueror` command with KDE 2 --- the implementation selects the best strategy for running Konqueror. @@ -163,6 +163,11 @@ Notes: (3) Only on macOS platform. +.. versionadded:: 3.2 + A new :class:`!MacOSXOSAScript` class has been added + and is used on Mac instead of the previous :class:`!MacOSX` class. + This adds support for opening browsers not currently set as the OS default. + .. versionadded:: 3.3 Support for Chrome/Chromium has been added. @@ -171,9 +176,6 @@ Notes: Removed browsers include Grail, Mosaic, Netscape, Galeon, Skipstone, Iceape, and Firefox versions 35 and below. -.. deprecated-removed:: 3.11 3.13 - :class:`MacOSX` is deprecated, use :class:`MacOSXOSAScript` instead. - Here are some simple examples:: url = 'https://docs.python.org/' @@ -194,7 +196,7 @@ Browser controllers provide these methods which parallel three of the module-level convenience functions: -.. attribute:: name +.. attribute:: controller.name System-dependent name for the browser. diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index 4ab671817710dd..06bd4d87eb03c6 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -288,7 +288,7 @@ This module offers the following functions: table (FAT) file system, the filename may not have an extension. A call to :func:`LoadKey` fails if the calling process does not have the - :const:`SE_RESTORE_PRIVILEGE` privilege. Note that privileges are different + :c:data:`!SE_RESTORE_PRIVILEGE` privilege. Note that privileges are different from permissions -- see the `RegLoadKey documentation `__ for more details. @@ -414,7 +414,7 @@ This module offers the following functions: If *key* represents a key on a remote computer, the path described by *file_name* is relative to the remote computer. The caller of this method must - possess the :const:`SeBackupPrivilege` security privilege. Note that + possess the **SeBackupPrivilege** security privilege. Note that privileges are different than permissions -- see the `Conflicts Between User Rights and Permissions documentation `__ @@ -536,7 +536,7 @@ This module offers the following functions: Constants ------------------ -The following constants are defined for use in many :mod:`_winreg` functions. +The following constants are defined for use in many :mod:`winreg` functions. .. _hkey-constants: @@ -745,7 +745,7 @@ All registry functions in this module return one of these objects. All registry functions in this module which accept a handle object also accept an integer, however, use of the handle object is encouraged. -Handle objects provide semantics for :meth:`__bool__` -- thus :: +Handle objects provide semantics for :meth:`~object.__bool__` -- thus :: if handle: print("Yes") diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 372f792a0f938e..370c5216652ba7 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -24,7 +24,7 @@ provided by Windows platforms. It includes functions and several constants. .. function:: PlaySound(sound, flags) - Call the underlying :c:func:`PlaySound` function from the Platform API. The + Call the underlying :c:func:`!PlaySound` function from the Platform API. The *sound* parameter may be a filename, a system sound alias, audio data as a :term:`bytes-like object`, or ``None``. Its interpretation depends on the value of *flags*, which can be a bitwise ORed @@ -35,7 +35,7 @@ provided by Windows platforms. It includes functions and several constants. .. function:: MessageBeep(type=MB_OK) - Call the underlying :c:func:`MessageBeep` function from the Platform API. This + Call the underlying :c:func:`!MessageBeep` function from the Platform API. This plays a sound as specified in the registry. The *type* argument specifies which sound to play; possible values are ``-1``, ``MB_ICONASTERISK``, ``MB_ICONEXCLAMATION``, ``MB_ICONHAND``, ``MB_ICONQUESTION``, and ``MB_OK``, all diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 39a4c1ba142338..be9e56b04c1fbf 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -180,7 +180,7 @@ also provides these miscellaneous utilities: print(chunk) .. versionchanged:: 3.11 - Support for :meth:`__getitem__` method has been removed. + Support for :meth:`~object.__getitem__` method has been removed. :mod:`wsgiref.headers` -- WSGI response header tools @@ -201,7 +201,7 @@ manipulation of WSGI response headers using a mapping-like interface. an empty list. :class:`Headers` objects support typical mapping operations including - :meth:`__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, + :meth:`~object.__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, :meth:`__delitem__` and :meth:`__contains__`. For each of these methods, the key is the header name (treated case-insensitively), and the value is the first value associated with that header name. Setting a header diff --git a/Doc/library/xdrlib.rst b/Doc/library/xdrlib.rst deleted file mode 100644 index 39e75573260c50..00000000000000 --- a/Doc/library/xdrlib.rst +++ /dev/null @@ -1,283 +0,0 @@ -:mod:`xdrlib` --- Encode and decode XDR data -============================================ - -.. module:: xdrlib - :synopsis: Encoders and decoders for the External Data Representation (XDR). - :deprecated: - -**Source code:** :source:`Lib/xdrlib.py` - -.. index:: - single: XDR - single: External Data Representation - -.. deprecated-removed:: 3.11 3.13 - The :mod:`xdrlib` module is deprecated - (see :pep:`PEP 594 <594#xdrlib>` for details). - --------------- - -The :mod:`xdrlib` module supports the External Data Representation Standard as -described in :rfc:`1014`, written by Sun Microsystems, Inc. June 1987. It -supports most of the data types described in the RFC. - -The :mod:`xdrlib` module defines two classes, one for packing variables into XDR -representation, and another for unpacking from XDR representation. There are -also two exception classes. - - -.. class:: Packer() - - :class:`Packer` is the class for packing data into XDR representation. The - :class:`Packer` class is instantiated with no arguments. - - -.. class:: Unpacker(data) - - ``Unpacker`` is the complementary class which unpacks XDR data values from a - string buffer. The input buffer is given as *data*. - - -.. seealso:: - - :rfc:`1014` - XDR: External Data Representation Standard - This RFC defined the encoding of data which was XDR at the time this module was - originally written. It has apparently been obsoleted by :rfc:`1832`. - - :rfc:`1832` - XDR: External Data Representation Standard - Newer RFC that provides a revised definition of XDR. - - -.. _xdr-packer-objects: - -Packer Objects --------------- - -:class:`Packer` instances have the following methods: - - -.. method:: Packer.get_buffer() - - Returns the current pack buffer as a string. - - -.. method:: Packer.reset() - - Resets the pack buffer to the empty string. - -In general, you can pack any of the most common XDR data types by calling the -appropriate ``pack_type()`` method. Each method takes a single argument, the -value to pack. The following simple data type packing methods are supported: -:meth:`pack_uint`, :meth:`pack_int`, :meth:`pack_enum`, :meth:`pack_bool`, -:meth:`pack_uhyper`, and :meth:`pack_hyper`. - - -.. method:: Packer.pack_float(value) - - Packs the single-precision floating point number *value*. - - -.. method:: Packer.pack_double(value) - - Packs the double-precision floating point number *value*. - -The following methods support packing strings, bytes, and opaque data: - - -.. method:: Packer.pack_fstring(n, s) - - Packs a fixed length string, *s*. *n* is the length of the string but it is - *not* packed into the data buffer. The string is padded with null bytes if - necessary to guaranteed 4 byte alignment. - - -.. method:: Packer.pack_fopaque(n, data) - - Packs a fixed length opaque data stream, similarly to :meth:`pack_fstring`. - - -.. method:: Packer.pack_string(s) - - Packs a variable length string, *s*. The length of the string is first packed - as an unsigned integer, then the string data is packed with - :meth:`pack_fstring`. - - -.. method:: Packer.pack_opaque(data) - - Packs a variable length opaque data string, similarly to :meth:`pack_string`. - - -.. method:: Packer.pack_bytes(bytes) - - Packs a variable length byte stream, similarly to :meth:`pack_string`. - -The following methods support packing arrays and lists: - - -.. method:: Packer.pack_list(list, pack_item) - - Packs a *list* of homogeneous items. This method is useful for lists with an - indeterminate size; i.e. the size is not available until the entire list has - been walked. For each item in the list, an unsigned integer ``1`` is packed - first, followed by the data value from the list. *pack_item* is the function - that is called to pack the individual item. At the end of the list, an unsigned - integer ``0`` is packed. - - For example, to pack a list of integers, the code might appear like this:: - - import xdrlib - p = xdrlib.Packer() - p.pack_list([1, 2, 3], p.pack_int) - - -.. method:: Packer.pack_farray(n, array, pack_item) - - Packs a fixed length list (*array*) of homogeneous items. *n* is the length of - the list; it is *not* packed into the buffer, but a :exc:`ValueError` exception - is raised if ``len(array)`` is not equal to *n*. As above, *pack_item* is the - function used to pack each element. - - -.. method:: Packer.pack_array(list, pack_item) - - Packs a variable length *list* of homogeneous items. First, the length of the - list is packed as an unsigned integer, then each element is packed as in - :meth:`pack_farray` above. - - -.. _xdr-unpacker-objects: - -Unpacker Objects ----------------- - -The :class:`Unpacker` class offers the following methods: - - -.. method:: Unpacker.reset(data) - - Resets the string buffer with the given *data*. - - -.. method:: Unpacker.get_position() - - Returns the current unpack position in the data buffer. - - -.. method:: Unpacker.set_position(position) - - Sets the data buffer unpack position to *position*. You should be careful about - using :meth:`get_position` and :meth:`set_position`. - - -.. method:: Unpacker.get_buffer() - - Returns the current unpack data buffer as a string. - - -.. method:: Unpacker.done() - - Indicates unpack completion. Raises an :exc:`Error` exception if all of the - data has not been unpacked. - -In addition, every data type that can be packed with a :class:`Packer`, can be -unpacked with an :class:`Unpacker`. Unpacking methods are of the form -``unpack_type()``, and take no arguments. They return the unpacked object. - - -.. method:: Unpacker.unpack_float() - - Unpacks a single-precision floating point number. - - -.. method:: Unpacker.unpack_double() - - Unpacks a double-precision floating point number, similarly to - :meth:`unpack_float`. - -In addition, the following methods unpack strings, bytes, and opaque data: - - -.. method:: Unpacker.unpack_fstring(n) - - Unpacks and returns a fixed length string. *n* is the number of characters - expected. Padding with null bytes to guaranteed 4 byte alignment is assumed. - - -.. method:: Unpacker.unpack_fopaque(n) - - Unpacks and returns a fixed length opaque data stream, similarly to - :meth:`unpack_fstring`. - - -.. method:: Unpacker.unpack_string() - - Unpacks and returns a variable length string. The length of the string is first - unpacked as an unsigned integer, then the string data is unpacked with - :meth:`unpack_fstring`. - - -.. method:: Unpacker.unpack_opaque() - - Unpacks and returns a variable length opaque data string, similarly to - :meth:`unpack_string`. - - -.. method:: Unpacker.unpack_bytes() - - Unpacks and returns a variable length byte stream, similarly to - :meth:`unpack_string`. - -The following methods support unpacking arrays and lists: - - -.. method:: Unpacker.unpack_list(unpack_item) - - Unpacks and returns a list of homogeneous items. The list is unpacked one - element at a time by first unpacking an unsigned integer flag. If the flag is - ``1``, then the item is unpacked and appended to the list. A flag of ``0`` - indicates the end of the list. *unpack_item* is the function that is called to - unpack the items. - - -.. method:: Unpacker.unpack_farray(n, unpack_item) - - Unpacks and returns (as a list) a fixed length array of homogeneous items. *n* - is number of list elements to expect in the buffer. As above, *unpack_item* is - the function used to unpack each element. - - -.. method:: Unpacker.unpack_array(unpack_item) - - Unpacks and returns a variable length *list* of homogeneous items. First, the - length of the list is unpacked as an unsigned integer, then each element is - unpacked as in :meth:`unpack_farray` above. - - -.. _xdr-exceptions: - -Exceptions ----------- - -Exceptions in this module are coded as class instances: - - -.. exception:: Error - - The base exception class. :exc:`Error` has a single public attribute - :attr:`msg` containing the description of the error. - - -.. exception:: ConversionError - - Class derived from :exc:`Error`. Contains no additional instance variables. - -Here is an example of how you would catch one of these exceptions:: - - import xdrlib - p = xdrlib.Packer() - try: - p.pack_double(8.01) - except xdrlib.ConversionError as instance: - print('packing the double failed:', instance.msg) - diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index d1df465a598e53..843c2fd7fdb937 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -115,7 +115,7 @@ DOMEventStream Objects .. class:: DOMEventStream(stream, parser, bufsize) .. versionchanged:: 3.11 - Support for :meth:`__getitem__` method has been removed. + Support for :meth:`~object.__getitem__` method has been removed. .. method:: getEvent() diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index f9290f528e5555..57cfbb8d92244b 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -17,7 +17,7 @@ for parsing and creating XML data. This module will use a fast implementation whenever available. .. deprecated:: 3.3 - The :mod:`xml.etree.cElementTree` module is deprecated. + The :mod:`!xml.etree.cElementTree` module is deprecated. .. warning:: @@ -154,6 +154,7 @@ elements, call :meth:`XMLPullParser.read_events`. Here is an example:: ... print(elem.tag, 'text=', elem.text) ... end + mytag text= sometext more text The obvious use case is applications that operate in a non-blocking fashion where the XML data is being received from a socket or read incrementally from @@ -621,7 +622,9 @@ Functions *parser* is an optional parser instance. If not given, the standard :class:`XMLParser` parser is used. *parser* must be a subclass of :class:`XMLParser` and can only use the default :class:`TreeBuilder` as a - target. Returns an :term:`iterator` providing ``(event, elem)`` pairs. + target. Returns an :term:`iterator` providing ``(event, elem)`` pairs; + it has a ``root`` attribute that references the root element of the + resulting XML tree once *source* is fully read. Note that while :func:`iterparse` builds the tree incrementally, it issues blocking reads on *source* (or the file it names). As such, it's unsuitable @@ -825,6 +828,8 @@ Reference Functions ^^^^^^^^^ +.. module:: xml.etree.ElementInclude + .. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None) :module: @@ -862,6 +867,9 @@ Functions Element Objects ^^^^^^^^^^^^^^^ +.. module:: xml.etree.ElementTree + :noindex: + .. class:: Element(tag, attrib={}, **extra) Element class. This class defines the Element interface, and provides a diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 20b0905bb1093a..909022ea4ba6a4 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -73,12 +73,12 @@ decompression bomb Safe Safe Safe 1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to potential reliance on system-provided libraries. Check - :data:`pyexpat.EXPAT_VERSION`. + :const:`!pyexpat.EXPAT_VERSION`. 2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a - :exc:`ParserError` when an entity occurs. + :exc:`~xml.etree.ElementTree.ParseError` when an entity occurs. 3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns the unexpanded entity verbatim. -4. :mod:`xmlrpclib` doesn't expand external entities and omits them. +4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. 5. Since Python 3.7.1, external general entities are no longer processed by default. @@ -119,8 +119,8 @@ all known attack vectors with examples and references. .. _defusedxml-package: -The :mod:`defusedxml` Package ------------------------------------------------------- +The :mod:`!defusedxml` Package +------------------------------ `defusedxml`_ is a pure Python package with modified subclasses of all stdlib XML parsers that prevent any potentially malicious operation. Use of this diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index 719ce5ab1bcf65..e2f28e3244cb09 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -393,7 +393,7 @@ implements this interface, then register the object with your :class:`~xml.sax.xmlreader.XMLReader`, the parser will call the methods in your object to report all warnings and errors. There are three levels of errors available: warnings, (possibly) recoverable errors, -and unrecoverable errors. All methods take a :exc:`SAXParseException` as the +and unrecoverable errors. All methods take a :exc:`~xml.sax.SAXParseException` as the only parameter. Errors and warnings may be converted to an exception by raising the passed-in exception object. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index ab4606bcf9fe6c..e57e76dcac7820 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -92,5 +92,5 @@ or as base classes. reading. The input source can be given as a string, a file-like object, or an :class:`~xml.sax.xmlreader.InputSource` object; parsers will use this function to implement the polymorphic *source* argument to their - :meth:`parse` method. + :meth:`~xml.sax.xmlreader.XMLReader.parse` method. diff --git a/Doc/library/xmlrpc.rst b/Doc/library/xmlrpc.rst index ae68157b0f63c1..5f0a2cf68d01f9 100644 --- a/Doc/library/xmlrpc.rst +++ b/Doc/library/xmlrpc.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc` --- XMLRPC server and client modules -================================================== +:mod:`!xmlrpc` --- XMLRPC server and client modules +=================================================== XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP as a transport. With it, a client can call methods with parameters on a remote diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 981020b13cd988..104afca23a20b4 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -54,7 +54,7 @@ The following options are understood: .. program:: zipapp -.. cmdoption:: -o , --output= +.. option:: -o , --output= Write the output to a file named *output*. If this option is not specified, the output filename will be the same as the input *source*, with the @@ -64,13 +64,13 @@ The following options are understood: An output filename must be specified if the *source* is an archive (and in that case, *output* must not be the same as *source*). -.. cmdoption:: -p , --python= +.. option:: -p , --python= Add a ``#!`` line to the archive specifying *interpreter* as the command to run. Also, on POSIX, make the archive executable. The default is to write no ``#!`` line, and not make the file executable. -.. cmdoption:: -m , --main= +.. option:: -m , --main= Write a ``__main__.py`` file to the archive that executes *mainfn*. The *mainfn* argument should have the form "pkg.mod:fn", where "pkg.mod" is a @@ -79,7 +79,7 @@ The following options are understood: :option:`--main` cannot be specified when copying an archive. -.. cmdoption:: -c, --compress +.. option:: -c, --compress Compress files with the deflate method, reducing the size of the output file. By default, files are stored uncompressed in the archive. @@ -88,13 +88,13 @@ The following options are understood: .. versionadded:: 3.7 -.. cmdoption:: --info +.. option:: --info Display the interpreter embedded in the archive, for diagnostic purposes. In this case, any other options are ignored and SOURCE must be an archive, not a directory. -.. cmdoption:: -h, --help +.. option:: -h, --help Print a short usage message and exit. @@ -281,12 +281,7 @@ The steps to create a standalone archive are as follows: file - if not, you can just list the dependencies manually on the pip command line). -3. Optionally, delete the ``.dist-info`` directories created by pip in the - ``myapp`` directory. These hold metadata for pip to manage the packages, and - as you won't be making any further use of pip they aren't required - - although it won't do any harm if you leave them. - -4. Package the application using: +3. Package the application using: .. code-block:: shell-session @@ -303,115 +298,18 @@ the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions when installed. -Making a Windows executable -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -On Windows, registration of the ``.pyz`` extension is optional, and -furthermore, there are certain places that don't recognise registered -extensions "transparently" (the simplest example is that -``subprocess.run(['myapp'])`` won't find your application - you need to -explicitly specify the extension). - -On Windows, therefore, it is often preferable to create an executable from the -zipapp. This is relatively easy, although it does require a C compiler. The -basic approach relies on the fact that zipfiles can have arbitrary data -prepended, and Windows exe files can have arbitrary data appended. So by -creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, -you end up with a single-file executable that runs your application. - -A suitable launcher can be as simple as the following:: - - #define Py_LIMITED_API 1 - #include "Python.h" - - #define WIN32_LEAN_AND_MEAN - #include - - #ifdef WINDOWS - int WINAPI wWinMain( - HINSTANCE hInstance, /* handle to current instance */ - HINSTANCE hPrevInstance, /* handle to previous instance */ - LPWSTR lpCmdLine, /* pointer to command line */ - int nCmdShow /* show state of window */ - ) - #else - int wmain() - #endif - { - wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); - myargv[0] = __wargv[0]; - memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); - return Py_Main(__argc+1, myargv); - } - -If you define the ``WINDOWS`` preprocessor symbol, this will generate a -GUI executable, and without it, a console executable. - -To compile the executable, you can either just use the standard MSVC -command line tools, or you can take advantage of the fact that distutils -knows how to compile Python source:: - - >>> from distutils.ccompiler import new_compiler - >>> import distutils.sysconfig - >>> import sys - >>> import os - >>> from pathlib import Path - - >>> def compile(src): - >>> src = Path(src) - >>> cc = new_compiler() - >>> exe = src.stem - >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) - >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) - >>> # First the CLI executable - >>> objs = cc.compile([str(src)]) - >>> cc.link_executable(objs, exe) - >>> # Now the GUI executable - >>> cc.define_macro('WINDOWS') - >>> objs = cc.compile([str(src)]) - >>> cc.link_executable(objs, exe + 'w') - - >>> if __name__ == "__main__": - >>> compile("zastub.c") - -The resulting launcher uses the "Limited ABI", so it will run unchanged with -any version of Python 3.x. All it needs is for Python (``python3.dll``) to be -on the user's ``PATH``. - -For a fully standalone distribution, you can distribute the launcher with your -application appended, bundled with the Python "embedded" distribution. This -will run on any PC with the appropriate architecture (32 bit or 64 bit). - - Caveats ~~~~~~~ -There are some limitations to the process of bundling your application into -a single file. In most, if not all, cases they can be addressed without -needing major changes to your application. - -1. If your application depends on a package that includes a C extension, that - package cannot be run from a zip file (this is an OS limitation, as executable - code must be present in the filesystem for the OS loader to load it). In this - case, you can exclude that dependency from the zipfile, and either require - your users to have it installed, or ship it alongside your zipfile and add code - to your ``__main__.py`` to include the directory containing the unzipped - module in ``sys.path``. In this case, you will need to make sure to ship - appropriate binaries for your target architecture(s) (and potentially pick the - correct version to add to ``sys.path`` at runtime, based on the user's machine). - -2. If you are shipping a Windows executable as described above, you either need to - ensure that your users have ``python3.dll`` on their PATH (which is not the - default behaviour of the installer) or you should bundle your application with - the embedded distribution. - -3. The suggested launcher above uses the Python embedding API. This means that in - your application, ``sys.executable`` will be your application, and *not* a - conventional Python interpreter. Your code and its dependencies need to be - prepared for this possibility. For example, if your application uses the - :mod:`multiprocessing` module, it will need to call - :func:`multiprocessing.set_executable` to let the module know where to find the - standard Python interpreter. +If your application depends on a package that includes a C extension, that +package cannot be run from a zip file (this is an OS limitation, as executable +code must be present in the filesystem for the OS loader to load it). In this +case, you can exclude that dependency from the zipfile, and either require +your users to have it installed, or ship it alongside your zipfile and add code +to your ``__main__.py`` to include the directory containing the unzipped +module in ``sys.path``. In this case, you will need to make sure to ship +appropriate binaries for your target architecture(s) (and potentially pick the +correct version to add to ``sys.path`` at runtime, based on the user's machine). The Python Zip Application Archive Format diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 45f3d340bd82d3..a77e49a7643826 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -577,7 +577,8 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. .. data:: Path.suffix - The file extension of the final component. + The last dot-separated portion of the final component, if any. + This is commonly called the file extension. .. versionadded:: 3.11 Added :data:`Path.suffix` property. @@ -591,7 +592,7 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. .. data:: Path.suffixes - A list of the path’s file extensions. + A list of the path’s suffixes, commonly called file extensions. .. versionadded:: 3.11 Added :data:`Path.suffixes` property. @@ -905,27 +906,27 @@ For a list of the files in a ZIP archive, use the :option:`-l` option: Command-line options ~~~~~~~~~~~~~~~~~~~~ -.. cmdoption:: -l - --list +.. option:: -l + --list List files in a zipfile. -.. cmdoption:: -c ... - --create ... +.. option:: -c ... + --create ... Create zipfile from source files. -.. cmdoption:: -e - --extract +.. option:: -e + --extract Extract zipfile into target directory. -.. cmdoption:: -t - --test +.. option:: -t + --test Test whether the zipfile is valid or not. -.. cmdoption:: --metadata-encoding +.. option:: --metadata-encoding Specify encoding of member names for :option:`-l`, :option:`-e` and :option:`-t`. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 11d19e8c863e9f..47c81f0e63603d 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -113,7 +113,7 @@ zipimporter Objects file wasn't found. .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. + :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. .. method:: get_filename(fullname) diff --git a/Doc/license.rst b/Doc/license.rst index 005d048b6eb209..8aad93062a5a88 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -352,8 +352,8 @@ the verbatim comments from the original code:: Sockets ------- -The :mod:`socket` module uses the functions, :func:`getaddrinfo`, and -:func:`getnameinfo`, which are coded in separate source files from the WIDE +The :mod:`socket` module uses the functions, :c:func:`!getaddrinfo`, and +:c:func:`!getnameinfo`, which are coded in separate source files from the WIDE Project, https://www.wide.ad.jp/. :: Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. @@ -387,7 +387,7 @@ Project, https://www.wide.ad.jp/. :: Asynchronous socket services ---------------------------- -The :mod:`test.support.asynchat` and :mod:`test.support.asyncore` +The :mod:`!test.support.asynchat` and :mod:`!test.support.asyncore` modules contain the following notice:: Copyright 1996 by Sam Rushing @@ -476,7 +476,7 @@ The :mod:`trace` module contains the following notice:: UUencode and UUdecode functions ------------------------------- -The :mod:`uu` module contains the following notice:: +The ``uu`` codec contains the following notice:: Copyright 1994 by Lance Ellinghouse Cathedral City, California Republic, United States of America. @@ -539,7 +539,7 @@ The :mod:`xmlrpc.client` module contains the following notice:: test_epoll ---------- -The :mod:`test_epoll` module contains the following notice:: +The :mod:`!test.test_epoll` module contains the following notice:: Copyright (c) 2001-2006 Twisted Matrix Laboratories. @@ -655,144 +655,196 @@ copyright and licensing notice:: OpenSSL ------- -The modules :mod:`hashlib`, :mod:`posix`, :mod:`ssl`, :mod:`crypt` use +The modules :mod:`hashlib`, :mod:`posix` and :mod:`ssl` use the OpenSSL library for added performance if made available by the operating system. Additionally, the Windows and macOS installers for Python may include a copy of the OpenSSL libraries, so we include a copy -of the OpenSSL license here:: - - - LICENSE ISSUES - ============== - - The OpenSSL toolkit stays under a dual license, i.e. both the conditions of - the OpenSSL License and the original SSLeay license apply to the toolkit. - See below for the actual license texts. Actually both licenses are BSD-style - Open Source licenses. In case of any license issues related to OpenSSL - please contact openssl-core@openssl.org. - - OpenSSL License - --------------- - - /* ==================================================================== - * Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * openssl-core@openssl.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.openssl.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - - Original SSLeay License - ----------------------- - - /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) - * All rights reserved. - * - * This package is an SSL implementation written - * by Eric Young (eay@cryptsoft.com). - * The implementation was written so as to conform with Netscapes SSL. - * - * This library is free for commercial and non-commercial use as long as - * the following conditions are aheared to. The following conditions - * apply to all code found in this distribution, be it the RC4, RSA, - * lhash, DES, etc., code; not just the SSL code. The SSL documentation - * included with this distribution is covered by the same copyright terms - * except that the holder is Tim Hudson (tjh@cryptsoft.com). - * - * Copyright remains Eric Young's, and as such any Copyright notices in - * the code are not to be removed. - * If this package is used in a product, Eric Young should be given attribution - * as the author of the parts of the library used. - * This can be in the form of a textual message at program startup or - * in documentation (online or textual) provided with the package. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * "This product includes cryptographic software written by - * Eric Young (eay@cryptsoft.com)" - * The word 'cryptographic' can be left out if the rouines from the library - * being used are not cryptographic related :-). - * 4. If you include any Windows specific code (or a derivative thereof) from - * the apps directory (application code) you must include an acknowledgement: - * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" - * - * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The licence and distribution terms for any publically available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution licence - * [including the GNU Public Licence.] - */ +of the OpenSSL license here. For the OpenSSL 3.0 release, +and later releases derived from that, the Apache License v2 applies:: + + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS expat ----- -The :mod:`pyexpat` extension is built using an included copy of the expat +The :mod:`pyexpat ` extension is built using an included copy of the expat sources unless the build is configured ``--with-system-expat``:: Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd @@ -990,29 +1042,27 @@ https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Audioop -------- +mimalloc +-------- + +MIT License + +Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -The audioop module uses the code base in g771.c file of the SoX project:: - - Programming the AdLib/Sound Blaster - FM Music Chips - Version 2.0 (24 Feb 1992) - Copyright (c) 1991, 1992 by Jeffrey S. Lee - jlee@smylex.uucp - Warranty and Copyright Policy - This document is provided on an "as-is" basis, and its author makes - no warranty or representation, express or implied, with respect to - its quality performance or fitness for a particular purpose. In no - event will the author of this document be liable for direct, indirect, - special, incidental, or consequential damages arising out of the use - or inability to use the information contained within. Use of this - document is at your own risk. - This file may be used and copied freely so long as the applicable - copyright notices are retained, and no modifications are made to the - text of the document. No money shall be charged for its distribution - beyond reasonable shipping, handling and duplication costs, nor shall - proprietary changes be made to this document so that it cannot be - distributed freely. This document may not be included in published - material or commercial packages without the written consent of its - author. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 9d1e5b6c596d9f..8f6481339837a0 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -489,37 +489,37 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo #. The context expression (the expression given in the :token:`~python-grammar:with_item`) is evaluated to obtain a context manager. -#. The context manager's :meth:`__enter__` is loaded for later use. +#. The context manager's :meth:`~object.__enter__` is loaded for later use. -#. The context manager's :meth:`__exit__` is loaded for later use. +#. The context manager's :meth:`~object.__exit__` is loaded for later use. -#. The context manager's :meth:`__enter__` method is invoked. +#. The context manager's :meth:`~object.__enter__` method is invoked. #. If a target was included in the :keyword:`with` statement, the return value - from :meth:`__enter__` is assigned to it. + from :meth:`~object.__enter__` is assigned to it. .. note:: - The :keyword:`with` statement guarantees that if the :meth:`__enter__` - method returns without an error, then :meth:`__exit__` will always be + The :keyword:`with` statement guarantees that if the :meth:`~object.__enter__` + method returns without an error, then :meth:`~object.__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would be. See step 7 below. #. The suite is executed. -#. The context manager's :meth:`__exit__` method is invoked. If an exception +#. The context manager's :meth:`~object.__exit__` method is invoked. If an exception caused the suite to be exited, its type, value, and traceback are passed as - arguments to :meth:`__exit__`. Otherwise, three :const:`None` arguments are + arguments to :meth:`~object.__exit__`. Otherwise, three :const:`None` arguments are supplied. If the suite was exited due to an exception, and the return value from the - :meth:`__exit__` method was false, the exception is reraised. If the return + :meth:`~object.__exit__` method was false, the exception is reraised. If the return value was true, the exception is suppressed, and execution continues with the statement following the :keyword:`with` statement. If the suite was exited for any reason other than an exception, the return - value from :meth:`__exit__` is ignored, and execution proceeds at the normal + value from :meth:`~object.__exit__` is ignored, and execution proceeds at the normal location for the kind of exit that was taken. The following code:: @@ -642,14 +642,14 @@ Here's an overview of the logical flow of a match statement: specified below. **Name bindings made during a successful pattern match outlive the executed block and can be used after the match statement**. - .. note:: + .. note:: - During failed pattern matches, some subpatterns may succeed. Do not - rely on bindings being made for a failed match. Conversely, do not - rely on variables remaining unchanged after a failed match. The exact - behavior is dependent on implementation and may vary. This is an - intentional decision made to allow different implementations to add - optimizations. + During failed pattern matches, some subpatterns may succeed. Do not + rely on bindings being made for a failed match. Conversely, do not + rely on variables remaining unchanged after a failed match. The exact + behavior is dependent on implementation and may vary. This is an + intentional decision made to allow different implementations to add + optimizations. #. If the pattern succeeds, the corresponding guard (if present) is evaluated. In this case all name bindings are guaranteed to have happened. @@ -1058,7 +1058,7 @@ subject value: .. note:: Key-value pairs are matched using the two-argument form of the mapping subject's ``get()`` method. Matched key-value pairs must already be present in the mapping, and not created on-the-fly via :meth:`__missing__` or - :meth:`__getitem__`. + :meth:`~object.__getitem__`. In simple terms ``{KEY1: P1, KEY2: P2, ... }`` matches only if all the following happens: @@ -1170,8 +1170,10 @@ In simple terms ``CLS(P1, attr=P2)`` matches only if the following happens: * ``isinstance(, CLS)`` * convert ``P1`` to a keyword pattern using ``CLS.__match_args__`` * For each keyword argument ``attr=P2``: - * ``hasattr(, "attr")`` - * ``P2`` matches ``.attr`` + + * ``hasattr(, "attr")`` + * ``P2`` matches ``.attr`` + * ... and so on for the corresponding keyword argument/pattern pair. .. seealso:: @@ -1206,7 +1208,7 @@ A function definition defines a user-defined function object (see section :ref:`types`): .. productionlist:: python-grammar - funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" + funcdef: [`decorators`] "def" `funcname` [`type_params`] "(" [`parameter_list`] ")" : ["->" `expression`] ":" `suite` decorators: `decorator`+ decorator: "@" `assignment_expression` NEWLINE @@ -1256,6 +1258,15 @@ except that the original function is not temporarily bound to the name ``func``. :token:`~python-grammar:assignment_expression`. Previously, the grammar was much more restrictive; see :pep:`614` for details. +A list of :ref:`type parameters ` may be given in square brackets +between the function's name and the opening parenthesis for its parameter list. +This indicates to static type checkers that the function is generic. At runtime, +the type parameters can be retrieved from the function's ``__type_params__`` +attribute. See :ref:`generic-functions` for more. + +.. versionchanged:: 3.12 + Type parameter lists are new in Python 3.12. + .. index:: triple: default; parameter; value single: argument; function definition @@ -1378,7 +1389,7 @@ Class definitions A class definition defines a class object (see section :ref:`types`): .. productionlist:: python-grammar - classdef: [`decorators`] "class" `classname` [`inheritance`] ":" `suite` + classdef: [`decorators`] "class" `classname` [`type_params`] [`inheritance`] ":" `suite` inheritance: "(" [`argument_list`] ")" classname: `identifier` @@ -1434,6 +1445,15 @@ decorators. The result is then bound to the class name. :token:`~python-grammar:assignment_expression`. Previously, the grammar was much more restrictive; see :pep:`614` for details. +A list of :ref:`type parameters ` may be given in square brackets +immediately after the class's name. +This indicates to static type checkers that the class is generic. At runtime, +the type parameters can be retrieved from the class's ``__type_params__`` +attribute. See :ref:`generic-classes` for more. + +.. versionchanged:: 3.12 + Type parameter lists are new in Python 3.12. + **Programmer's note:** Variables defined in the class definition are class attributes; they are shared by instances. Instance attributes can be set in a method with ``self.name = value``. Both class and instance attributes are @@ -1589,6 +1609,228 @@ body of a coroutine function. The proposal that made coroutines a proper standalone concept in Python, and added supporting syntax. +.. _type-params: + +Type parameter lists +==================== + +.. versionadded:: 3.12 + +.. index:: + single: type parameters + +.. productionlist:: python-grammar + type_params: "[" `type_param` ("," `type_param`)* "]" + type_param: `typevar` | `typevartuple` | `paramspec` + typevar: `identifier` (":" `expression`)? + typevartuple: "*" `identifier` + paramspec: "**" `identifier` + +:ref:`Functions ` (including :ref:`coroutines `), +:ref:`classes ` and :ref:`type aliases ` may +contain a type parameter list:: + + def max[T](args: list[T]) -> T: + ... + + async def amax[T](args: list[T]) -> T: + ... + + class Bag[T]: + def __iter__(self) -> Iterator[T]: + ... + + def add(self, arg: T) -> None: + ... + + type ListOrSet[T] = list[T] | set[T] + +Semantically, this indicates that the function, class, or type alias is +generic over a type variable. This information is primarily used by static +type checkers, and at runtime, generic objects behave much like their +non-generic counterparts. + +Type parameters are declared in square brackets (``[]``) immediately +after the name of the function, class, or type alias. The type parameters +are accessible within the scope of the generic object, but not elsewhere. +Thus, after a declaration ``def func[T](): pass``, the name ``T`` is not available in +the module scope. Below, the semantics of generic objects are described +with more precision. The scope of type parameters is modeled with a special +function (technically, an :ref:`annotation scope `) that +wraps the creation of the generic object. + +Generic functions, classes, and type aliases have a :attr:`!__type_params__` +attribute listing their type parameters. + +Type parameters come in three kinds: + +* :data:`typing.TypeVar`, introduced by a plain name (e.g., ``T``). Semantically, this + represents a single type to a type checker. +* :data:`typing.TypeVarTuple`, introduced by a name prefixed with a single + asterisk (e.g., ``*Ts``). Semantically, this stands for a tuple of any + number of types. +* :data:`typing.ParamSpec`, introduced by a name prefixed with two asterisks + (e.g., ``**P``). Semantically, this stands for the parameters of a callable. + +:data:`typing.TypeVar` declarations can define *bounds* and *constraints* with +a colon (``:``) followed by an expression. A single expression after the colon +indicates a bound (e.g. ``T: int``). Semantically, this means +that the :data:`!typing.TypeVar` can only represent types that are a subtype of +this bound. A parenthesized tuple of expressions after the colon indicates a +set of constraints (e.g. ``T: (str, bytes)``). Each member of the tuple should be a +type (again, this is not enforced at runtime). Constrained type variables can only +take on one of the types in the list of constraints. + +For :data:`!typing.TypeVar`\ s declared using the type parameter list syntax, +the bound and constraints are not evaluated when the generic object is created, +but only when the value is explicitly accessed through the attributes ``__bound__`` +and ``__constraints__``. To accomplish this, the bounds or constraints are +evaluated in a separate :ref:`annotation scope `. + +:data:`typing.TypeVarTuple`\ s and :data:`typing.ParamSpec`\ s cannot have bounds +or constraints. + +The following example indicates the full set of allowed type parameter declarations:: + + def overly_generic[ + SimpleTypeVar, + TypeVarWithBound: int, + TypeVarWithConstraints: (str, bytes), + *SimpleTypeVarTuple, + **SimpleParamSpec, + ]( + a: SimpleTypeVar, + b: TypeVarWithBound, + c: Callable[SimpleParamSpec, TypeVarWithConstraints], + *d: SimpleTypeVarTuple, + ): ... + +.. _generic-functions: + +Generic functions +----------------- + +Generic functions are declared as follows:: + + def func[T](arg: T): ... + +This syntax is equivalent to:: + + annotation-def TYPE_PARAMS_OF_func(): + T = typing.TypeVar("T") + def func(arg: T): ... + func.__type_params__ = (T,) + return func + func = TYPE_PARAMS_OF_func() + +Here ``annotation-def`` indicates an :ref:`annotation scope `, +which is not actually bound to any name at runtime. (One +other liberty is taken in the translation: the syntax does not go through +attribute access on the :mod:`typing` module, but creates an instance of +:data:`typing.TypeVar` directly.) + +The annotations of generic functions are evaluated within the annotation scope +used for declaring the type parameters, but the function's defaults and +decorators are not. + +The following example illustrates the scoping rules for these cases, +as well as for additional flavors of type parameters:: + + @decorator + def func[T: int, *Ts, **P](*args: *Ts, arg: Callable[P, T] = some_default): + ... + +Except for the :ref:`lazy evaluation ` of the +:class:`~typing.TypeVar` bound, this is equivalent to:: + + DEFAULT_OF_arg = some_default + + annotation-def TYPE_PARAMS_OF_func(): + + annotation-def BOUND_OF_T(): + return int + # In reality, BOUND_OF_T() is evaluated only on demand. + T = typing.TypeVar("T", bound=BOUND_OF_T()) + + Ts = typing.TypeVarTuple("Ts") + P = typing.ParamSpec("P") + + def func(*args: *Ts, arg: Callable[P, T] = DEFAULT_OF_arg): + ... + + func.__type_params__ = (T, Ts, P) + return func + func = decorator(TYPE_PARAMS_OF_func()) + +The capitalized names like ``DEFAULT_OF_arg`` are not actually +bound at runtime. + +.. _generic-classes: + +Generic classes +--------------- + +Generic classes are declared as follows:: + + class Bag[T]: ... + +This syntax is equivalent to:: + + annotation-def TYPE_PARAMS_OF_Bag(): + T = typing.TypeVar("T") + class Bag(typing.Generic[T]): + __type_params__ = (T,) + ... + return Bag + Bag = TYPE_PARAMS_OF_Bag() + +Here again ``annotation-def`` (not a real keyword) indicates an +:ref:`annotation scope `, and the name +``TYPE_PARAMS_OF_Bag`` is not actually bound at runtime. + +Generic classes implicitly inherit from :data:`typing.Generic`. +The base classes and keyword arguments of generic classes are +evaluated within the type scope for the type parameters, +and decorators are evaluated outside that scope. This is illustrated +by this example:: + + @decorator + class Bag(Base[T], arg=T): ... + +This is equivalent to:: + + annotation-def TYPE_PARAMS_OF_Bag(): + T = typing.TypeVar("T") + class Bag(Base[T], typing.Generic[T], arg=T): + __type_params__ = (T,) + ... + return Bag + Bag = decorator(TYPE_PARAMS_OF_Bag()) + +.. _generic-type-aliases: + +Generic type aliases +-------------------- + +The :keyword:`type` statement can also be used to create a generic type alias:: + + type ListOrSet[T] = list[T] | set[T] + +Except for the :ref:`lazy evaluation ` of the value, +this is equivalent to:: + + annotation-def TYPE_PARAMS_OF_ListOrSet(): + T = typing.TypeVar("T") + + annotation-def VALUE_OF_ListOrSet(): + return list[T] | set[T] + # In reality, the value is lazily evaluated + return typing.TypeAliasType("ListOrSet", VALUE_OF_ListOrSet(), type_params=(T,)) + ListOrSet = TYPE_PARAMS_OF_ListOrSet() + +Here, ``annotation-def`` (not a real keyword) indicates an +:ref:`annotation scope `. The capitalized names +like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. .. rubric:: Footnotes @@ -1598,29 +1840,29 @@ body of a coroutine function. .. [#] In pattern matching, a sequence is defined as one of the following: - * a class that inherits from :class:`collections.abc.Sequence` - * a Python class that has been registered as :class:`collections.abc.Sequence` - * a builtin class that has its (CPython) :data:`Py_TPFLAGS_SEQUENCE` bit set - * a class that inherits from any of the above + * a class that inherits from :class:`collections.abc.Sequence` + * a Python class that has been registered as :class:`collections.abc.Sequence` + * a builtin class that has its (CPython) :c:macro:`Py_TPFLAGS_SEQUENCE` bit set + * a class that inherits from any of the above The following standard library classes are sequences: - * :class:`array.array` - * :class:`collections.deque` - * :class:`list` - * :class:`memoryview` - * :class:`range` - * :class:`tuple` + * :class:`array.array` + * :class:`collections.deque` + * :class:`list` + * :class:`memoryview` + * :class:`range` + * :class:`tuple` .. note:: Subject values of type ``str``, ``bytes``, and ``bytearray`` do not match sequence patterns. .. [#] In pattern matching, a mapping is defined as one of the following: - * a class that inherits from :class:`collections.abc.Mapping` - * a Python class that has been registered as :class:`collections.abc.Mapping` - * a builtin class that has its (CPython) :data:`Py_TPFLAGS_MAPPING` bit set - * a class that inherits from any of the above + * a class that inherits from :class:`collections.abc.Mapping` + * a Python class that has been registered as :class:`collections.abc.Mapping` + * a builtin class that has its (CPython) :c:macro:`Py_TPFLAGS_MAPPING` bit set + * a class that inherits from any of the above The standard library classes :class:`dict` and :class:`types.MappingProxyType` are mappings. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index c0734e49f29192..27d379a8b70f31 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -88,7 +88,7 @@ Some objects contain references to "external" resources such as open files or windows. It is understood that these resources are freed when the object is garbage-collected, but since garbage collection is not guaranteed to happen, such objects also provide an explicit way to release the external resource, -usually a :meth:`close` method. Programs are strongly recommended to explicitly +usually a :meth:`!close` method. Programs are strongly recommended to explicitly close such objects. The ':keyword:`try`...\ :keyword:`finally`' statement and the ':keyword:`with`' statement provide convenient ways to do this. @@ -141,1083 +141,1323 @@ Some of the type descriptions below contain a paragraph listing 'special attributes.' These are attributes that provide access to the implementation and are not intended for general use. Their definition may change in the future. + None - .. index:: pair: object; None +---- + +.. index:: pair: object; None + +This type has a single value. There is a single object with this value. This +object is accessed through the built-in name ``None``. It is used to signify the +absence of a value in many situations, e.g., it is returned from functions that +don't explicitly return anything. Its truth value is false. - This type has a single value. There is a single object with this value. This - object is accessed through the built-in name ``None``. It is used to signify the - absence of a value in many situations, e.g., it is returned from functions that - don't explicitly return anything. Its truth value is false. NotImplemented - .. index:: pair: object; NotImplemented +-------------- - This type has a single value. There is a single object with this value. This - object is accessed through the built-in name ``NotImplemented``. Numeric methods - and rich comparison methods should return this value if they do not implement the - operation for the operands provided. (The interpreter will then try the - reflected operation, or some other fallback, depending on the operator.) It - should not be evaluated in a boolean context. +.. index:: pair: object; NotImplemented - See - :ref:`implementing-the-arithmetic-operations` - for more details. +This type has a single value. There is a single object with this value. This +object is accessed through the built-in name ``NotImplemented``. Numeric methods +and rich comparison methods should return this value if they do not implement the +operation for the operands provided. (The interpreter will then try the +reflected operation, or some other fallback, depending on the operator.) It +should not be evaluated in a boolean context. - .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. +See +:ref:`implementing-the-arithmetic-operations` +for more details. + +.. versionchanged:: 3.9 + Evaluating ``NotImplemented`` in a boolean context is deprecated. While + it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. + It will raise a :exc:`TypeError` in a future version of Python. Ellipsis - .. index:: - pair: object; Ellipsis - single: ...; ellipsis literal +-------- +.. index:: + pair: object; Ellipsis + single: ...; ellipsis literal + +This type has a single value. There is a single object with this value. This +object is accessed through the literal ``...`` or the built-in name +``Ellipsis``. Its truth value is true. - This type has a single value. There is a single object with this value. This - object is accessed through the literal ``...`` or the built-in name - ``Ellipsis``. Its truth value is true. :class:`numbers.Number` - .. index:: pair: object; numeric +----------------------- + +.. index:: pair: object; numeric - These are created by numeric literals and returned as results by arithmetic - operators and arithmetic built-in functions. Numeric objects are immutable; - once created their value never changes. Python numbers are of course strongly - related to mathematical numbers, but subject to the limitations of numerical - representation in computers. +These are created by numeric literals and returned as results by arithmetic +operators and arithmetic built-in functions. Numeric objects are immutable; +once created their value never changes. Python numbers are of course strongly +related to mathematical numbers, but subject to the limitations of numerical +representation in computers. - The string representations of the numeric classes, computed by - :meth:`~object.__repr__` and :meth:`~object.__str__`, have the following - properties: +The string representations of the numeric classes, computed by +:meth:`~object.__repr__` and :meth:`~object.__str__`, have the following +properties: - * They are valid numeric literals which, when passed to their - class constructor, produce an object having the value of the - original numeric. +* They are valid numeric literals which, when passed to their + class constructor, produce an object having the value of the + original numeric. + +* The representation is in base 10, when possible. + +* Leading zeros, possibly excepting a single zero before a + decimal point, are not shown. + +* Trailing zeros, possibly excepting a single zero after a + decimal point, are not shown. + +* A sign is shown only when the number is negative. + +Python distinguishes between integers, floating point numbers, and complex +numbers: + + +:class:`numbers.Integral` +^^^^^^^^^^^^^^^^^^^^^^^^^ - * The representation is in base 10, when possible. +.. index:: pair: object; integer - * Leading zeros, possibly excepting a single zero before a - decimal point, are not shown. +These represent elements from the mathematical set of integers (positive and +negative). - * Trailing zeros, possibly excepting a single zero after a - decimal point, are not shown. +.. note:: + .. index:: pair: integer; representation - * A sign is shown only when the number is negative. + The rules for integer representation are intended to give the most meaningful + interpretation of shift and mask operations involving negative integers. - Python distinguishes between integers, floating point numbers, and complex - numbers: +There are two types of integers: - :class:`numbers.Integral` - .. index:: pair: object; integer +Integers (:class:`int`) + These represent numbers in an unlimited range, subject to available (virtual) + memory only. For the purpose of shift and mask operations, a binary + representation is assumed, and negative numbers are represented in a variant of + 2's complement which gives the illusion of an infinite string of sign bits + extending to the left. - These represent elements from the mathematical set of integers (positive and - negative). +Booleans (:class:`bool`) + .. index:: + pair: object; Boolean + single: False + single: True - There are two types of integers: + These represent the truth values False and True. The two objects representing + the values ``False`` and ``True`` are the only Boolean objects. The Boolean type is a + subtype of the integer type, and Boolean values behave like the values 0 and 1, + respectively, in almost all contexts, the exception being that when converted to + a string, the strings ``"False"`` or ``"True"`` are returned, respectively. - Integers (:class:`int`) - These represent numbers in an unlimited range, subject to available (virtual) - memory only. For the purpose of shift and mask operations, a binary - representation is assumed, and negative numbers are represented in a variant of - 2's complement which gives the illusion of an infinite string of sign bits - extending to the left. - Booleans (:class:`bool`) - .. index:: - pair: object; Boolean - single: False - single: True +:class:`numbers.Real` (:class:`float`) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - These represent the truth values False and True. The two objects representing - the values ``False`` and ``True`` are the only Boolean objects. The Boolean type is a - subtype of the integer type, and Boolean values behave like the values 0 and 1, - respectively, in almost all contexts, the exception being that when converted to - a string, the strings ``"False"`` or ``"True"`` are returned, respectively. +.. index:: + pair: object; floating point + pair: floating point; number + pair: C; language + pair: Java; language - .. index:: pair: integer; representation +These represent machine-level double precision floating point numbers. You are +at the mercy of the underlying machine architecture (and C or Java +implementation) for the accepted range and handling of overflow. Python does not +support single-precision floating point numbers; the savings in processor and +memory usage that are usually the reason for using these are dwarfed by the +overhead of using objects in Python, so there is no reason to complicate the +language with two kinds of floating point numbers. - The rules for integer representation are intended to give the most meaningful - interpretation of shift and mask operations involving negative integers. - :class:`numbers.Real` (:class:`float`) - .. index:: - pair: object; floating point - pair: floating point; number - pair: C; language - pair: Java; language +:class:`numbers.Complex` (:class:`complex`) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - These represent machine-level double precision floating point numbers. You are - at the mercy of the underlying machine architecture (and C or Java - implementation) for the accepted range and handling of overflow. Python does not - support single-precision floating point numbers; the savings in processor and - memory usage that are usually the reason for using these are dwarfed by the - overhead of using objects in Python, so there is no reason to complicate the - language with two kinds of floating point numbers. +.. index:: + pair: object; complex + pair: complex; number - :class:`numbers.Complex` (:class:`complex`) - .. index:: - pair: object; complex - pair: complex; number +These represent complex numbers as a pair of machine-level double precision +floating point numbers. The same caveats apply as for floating point numbers. +The real and imaginary parts of a complex number ``z`` can be retrieved through +the read-only attributes ``z.real`` and ``z.imag``. - These represent complex numbers as a pair of machine-level double precision - floating point numbers. The same caveats apply as for floating point numbers. - The real and imaginary parts of a complex number ``z`` can be retrieved through - the read-only attributes ``z.real`` and ``z.imag``. Sequences +--------- + +.. index:: + pair: built-in function; len + pair: object; sequence + single: index operation + single: item selection + single: subscription + +These represent finite ordered sets indexed by non-negative numbers. The +built-in function :func:`len` returns the number of items of a sequence. When +the length of a sequence is *n*, the index set contains the numbers 0, 1, +..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. + +.. index:: single: slicing + +Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such +that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a +sequence of the same type. This implies that the index set is renumbered so +that it starts at 0. + +Some sequences also support "extended slicing" with a third "step" parameter: +``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* +``>=`` ``0`` and *i* ``<=`` *x* ``<`` *j*. + +Sequences are distinguished according to their mutability: + + +Immutable sequences +^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; immutable sequence + pair: object; immutable + +An object of an immutable sequence type cannot change once it is created. (If +the object contains references to other objects, these other objects may be +mutable and may be changed; however, the collection of objects directly +referenced by an immutable object cannot change.) + +The following types are immutable sequences: + +.. index:: + single: string; immutable sequences + +Strings .. index:: - pair: built-in function; len - pair: object; sequence - single: index operation - single: item selection - single: subscription - - These represent finite ordered sets indexed by non-negative numbers. The - built-in function :func:`len` returns the number of items of a sequence. When - the length of a sequence is *n*, the index set contains the numbers 0, 1, - ..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. - - .. index:: single: slicing - - Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such - that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a - sequence of the same type. This implies that the index set is renumbered so - that it starts at 0. - - Some sequences also support "extended slicing" with a third "step" parameter: - ``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* - ``>=`` ``0`` and *i* ``<=`` *x* ``<`` *j*. - - Sequences are distinguished according to their mutability: - - Immutable sequences - .. index:: - pair: object; immutable sequence - pair: object; immutable - - An object of an immutable sequence type cannot change once it is created. (If - the object contains references to other objects, these other objects may be - mutable and may be changed; however, the collection of objects directly - referenced by an immutable object cannot change.) - - The following types are immutable sequences: - - .. index:: - single: string; immutable sequences - - Strings - .. index:: - pair: built-in function; chr - pair: built-in function; ord - single: character - single: integer - single: Unicode - - A string is a sequence of values that represent Unicode code points. - All the code points in the range ``U+0000 - U+10FFFF`` can be - represented in a string. Python doesn't have a :c:expr:`char` type; - instead, every code point in the string is represented as a string - object with length ``1``. The built-in function :func:`ord` - converts a code point from its string form to an integer in the - range ``0 - 10FFFF``; :func:`chr` converts an integer in the range - ``0 - 10FFFF`` to the corresponding length ``1`` string object. - :meth:`str.encode` can be used to convert a :class:`str` to - :class:`bytes` using the given text encoding, and - :meth:`bytes.decode` can be used to achieve the opposite. - - Tuples - .. index:: - pair: object; tuple - pair: singleton; tuple - pair: empty; tuple - - The items of a tuple are arbitrary Python objects. Tuples of two or - more items are formed by comma-separated lists of expressions. A tuple - of one item (a 'singleton') can be formed by affixing a comma to an - expression (an expression by itself does not create a tuple, since - parentheses must be usable for grouping of expressions). An empty - tuple can be formed by an empty pair of parentheses. - - Bytes - .. index:: bytes, byte - - A bytes object is an immutable array. The items are 8-bit bytes, - represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in :func:`bytes()` constructor - can be used to create bytes objects. Also, bytes objects can be - decoded to strings via the :meth:`~bytes.decode` method. - - Mutable sequences - .. index:: - pair: object; mutable sequence - pair: object; mutable - pair: assignment; statement - single: subscription - single: slicing - - Mutable sequences can be changed after they are created. The subscription and - slicing notations can be used as the target of assignment and :keyword:`del` - (delete) statements. - - There are currently two intrinsic mutable sequence types: - - Lists - .. index:: pair: object; list - - The items of a list are arbitrary Python objects. Lists are formed by - placing a comma-separated list of expressions in square brackets. (Note - that there are no special cases needed to form lists of length 0 or 1.) - - Byte Arrays - .. index:: bytearray - - A bytearray object is a mutable array. They are created by the built-in - :func:`bytearray` constructor. Aside from being mutable - (and hence unhashable), byte arrays otherwise provide the same interface - and functionality as immutable :class:`bytes` objects. - - .. index:: pair: module; array - - The extension module :mod:`array` provides an additional example of a - mutable sequence type, as does the :mod:`collections` module. + pair: built-in function; chr + pair: built-in function; ord + single: character + single: integer + single: Unicode + + A string is a sequence of values that represent Unicode code points. + All the code points in the range ``U+0000 - U+10FFFF`` can be + represented in a string. Python doesn't have a :c:expr:`char` type; + instead, every code point in the string is represented as a string + object with length ``1``. The built-in function :func:`ord` + converts a code point from its string form to an integer in the + range ``0 - 10FFFF``; :func:`chr` converts an integer in the range + ``0 - 10FFFF`` to the corresponding length ``1`` string object. + :meth:`str.encode` can be used to convert a :class:`str` to + :class:`bytes` using the given text encoding, and + :meth:`bytes.decode` can be used to achieve the opposite. + +Tuples + .. index:: + pair: object; tuple + pair: singleton; tuple + pair: empty; tuple + + The items of a tuple are arbitrary Python objects. Tuples of two or + more items are formed by comma-separated lists of expressions. A tuple + of one item (a 'singleton') can be formed by affixing a comma to an + expression (an expression by itself does not create a tuple, since + parentheses must be usable for grouping of expressions). An empty + tuple can be formed by an empty pair of parentheses. + +Bytes + .. index:: bytes, byte + + A bytes object is an immutable array. The items are 8-bit bytes, + represented by integers in the range 0 <= x < 256. Bytes literals + (like ``b'abc'``) and the built-in :func:`bytes()` constructor + can be used to create bytes objects. Also, bytes objects can be + decoded to strings via the :meth:`~bytes.decode` method. + + +Mutable sequences +^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; mutable sequence + pair: object; mutable + pair: assignment; statement + single: subscription + single: slicing + +Mutable sequences can be changed after they are created. The subscription and +slicing notations can be used as the target of assignment and :keyword:`del` +(delete) statements. + +.. note:: + .. index:: pair: module; array + .. index:: pair: module; collections + + The :mod:`collections` and :mod:`array` module provide + additional examples of mutable sequence types. + +There are currently two intrinsic mutable sequence types: + +Lists + .. index:: pair: object; list + + The items of a list are arbitrary Python objects. Lists are formed by + placing a comma-separated list of expressions in square brackets. (Note + that there are no special cases needed to form lists of length 0 or 1.) + +Byte Arrays + .. index:: bytearray + + A bytearray object is a mutable array. They are created by the built-in + :func:`bytearray` constructor. Aside from being mutable + (and hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable :class:`bytes` objects. + Set types - .. index:: - pair: built-in function; len - pair: object; set type +--------- - These represent unordered, finite sets of unique, immutable objects. As such, - they cannot be indexed by any subscript. However, they can be iterated over, and - the built-in function :func:`len` returns the number of items in a set. Common - uses for sets are fast membership testing, removing duplicates from a sequence, - and computing mathematical operations such as intersection, union, difference, - and symmetric difference. +.. index:: + pair: built-in function; len + pair: object; set type + +These represent unordered, finite sets of unique, immutable objects. As such, +they cannot be indexed by any subscript. However, they can be iterated over, and +the built-in function :func:`len` returns the number of items in a set. Common +uses for sets are fast membership testing, removing duplicates from a sequence, +and computing mathematical operations such as intersection, union, difference, +and symmetric difference. + +For set elements, the same immutability rules apply as for dictionary keys. Note +that numeric types obey the normal rules for numeric comparison: if two numbers +compare equal (e.g., ``1`` and ``1.0``), only one of them can be contained in a +set. + +There are currently two intrinsic set types: - For set elements, the same immutability rules apply as for dictionary keys. Note - that numeric types obey the normal rules for numeric comparison: if two numbers - compare equal (e.g., ``1`` and ``1.0``), only one of them can be contained in a - set. - There are currently two intrinsic set types: +Sets + .. index:: pair: object; set - Sets - .. index:: pair: object; set + These represent a mutable set. They are created by the built-in :func:`set` + constructor and can be modified afterwards by several methods, such as + :meth:`~set.add`. - These represent a mutable set. They are created by the built-in :func:`set` - constructor and can be modified afterwards by several methods, such as - :meth:`~set.add`. - Frozen sets - .. index:: pair: object; frozenset +Frozen sets + .. index:: pair: object; frozenset + + These represent an immutable set. They are created by the built-in + :func:`frozenset` constructor. As a frozenset is immutable and + :term:`hashable`, it can be used again as an element of another set, or as + a dictionary key. - These represent an immutable set. They are created by the built-in - :func:`frozenset` constructor. As a frozenset is immutable and - :term:`hashable`, it can be used again as an element of another set, or as - a dictionary key. Mappings - .. index:: - pair: built-in function; len - single: subscription - pair: object; mapping - - These represent finite sets of objects indexed by arbitrary index sets. The - subscript notation ``a[k]`` selects the item indexed by ``k`` from the mapping - ``a``; this can be used in expressions and as the target of assignments or - :keyword:`del` statements. The built-in function :func:`len` returns the number - of items in a mapping. - - There is currently a single intrinsic mapping type: - - Dictionaries - .. index:: pair: object; dictionary - - These represent finite sets of objects indexed by nearly arbitrary values. The - only types of values not acceptable as keys are values containing lists or - dictionaries or other mutable types that are compared by value rather than by - object identity, the reason being that the efficient implementation of - dictionaries requires a key's hash value to remain constant. Numeric types used - for keys obey the normal rules for numeric comparison: if two numbers compare - equal (e.g., ``1`` and ``1.0``) then they can be used interchangeably to index - the same dictionary entry. - - Dictionaries preserve insertion order, meaning that keys will be produced - in the same order they were added sequentially over the dictionary. - Replacing an existing key does not change the order, however removing a key - and re-inserting it will add it to the end instead of keeping its old place. - - Dictionaries are mutable; they can be created by the ``{...}`` notation (see - section :ref:`dict`). - - .. index:: - pair: module; dbm.ndbm - pair: module; dbm.gnu - - The extension modules :mod:`dbm.ndbm` and :mod:`dbm.gnu` provide - additional examples of mapping types, as does the :mod:`collections` - module. - - .. versionchanged:: 3.7 - Dictionaries did not preserve insertion order in versions of Python before 3.6. - In CPython 3.6, insertion order was preserved, but it was considered - an implementation detail at that time rather than a language guarantee. +-------- + +.. index:: + pair: built-in function; len + single: subscription + pair: object; mapping + +These represent finite sets of objects indexed by arbitrary index sets. The +subscript notation ``a[k]`` selects the item indexed by ``k`` from the mapping +``a``; this can be used in expressions and as the target of assignments or +:keyword:`del` statements. The built-in function :func:`len` returns the number +of items in a mapping. + +There is currently a single intrinsic mapping type: + + +Dictionaries +^^^^^^^^^^^^ + +.. index:: pair: object; dictionary + +These represent finite sets of objects indexed by nearly arbitrary values. The +only types of values not acceptable as keys are values containing lists or +dictionaries or other mutable types that are compared by value rather than by +object identity, the reason being that the efficient implementation of +dictionaries requires a key's hash value to remain constant. Numeric types used +for keys obey the normal rules for numeric comparison: if two numbers compare +equal (e.g., ``1`` and ``1.0``) then they can be used interchangeably to index +the same dictionary entry. + +Dictionaries preserve insertion order, meaning that keys will be produced +in the same order they were added sequentially over the dictionary. +Replacing an existing key does not change the order, however removing a key +and re-inserting it will add it to the end instead of keeping its old place. + +Dictionaries are mutable; they can be created by the ``{...}`` notation (see +section :ref:`dict`). + +.. index:: + pair: module; dbm.ndbm + pair: module; dbm.gnu + +The extension modules :mod:`dbm.ndbm` and :mod:`dbm.gnu` provide +additional examples of mapping types, as does the :mod:`collections` +module. + +.. versionchanged:: 3.7 + Dictionaries did not preserve insertion order in versions of Python before 3.6. + In CPython 3.6, insertion order was preserved, but it was considered + an implementation detail at that time rather than a language guarantee. + Callable types - .. index:: - pair: object; callable - pair: function; call - single: invocation - pair: function; argument - - These are the types to which the function call operation (see section - :ref:`calls`) can be applied: - - User-defined functions - .. index:: - pair: user-defined; function - pair: object; function - pair: object; user-defined function - - A user-defined function object is created by a function definition (see - section :ref:`function`). It should be called with an argument list - containing the same number of items as the function's formal parameter - list. - - Special attributes: - - .. tabularcolumns:: |l|L|l| - - .. index:: - single: __doc__ (function attribute) - single: __name__ (function attribute) - single: __module__ (function attribute) - single: __dict__ (function attribute) - single: __defaults__ (function attribute) - single: __closure__ (function attribute) - single: __code__ (function attribute) - single: __globals__ (function attribute) - single: __annotations__ (function attribute) - single: __kwdefaults__ (function attribute) - pair: global; namespace - - +-------------------------+-------------------------------+-----------+ - | Attribute | Meaning | | - +=========================+===============================+===========+ - | :attr:`__doc__` | The function's documentation | Writable | - | | string, or ``None`` if | | - | | unavailable; not inherited by | | - | | subclasses. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~definition.\ | The function's name. | Writable | - | __name__` | | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~definition.\ | The function's | Writable | - | __qualname__` | :term:`qualified name`. | | - | | | | - | | .. versionadded:: 3.3 | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__module__` | The name of the module the | Writable | - | | function was defined in, or | | - | | ``None`` if unavailable. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__defaults__` | A tuple containing default | Writable | - | | argument values for those | | - | | arguments that have defaults, | | - | | or ``None`` if no arguments | | - | | have a default value. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__code__` | The code object representing | Writable | - | | the compiled function body. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__globals__` | A reference to the dictionary | Read-only | - | | that holds the function's | | - | | global variables --- the | | - | | global namespace of the | | - | | module in which the function | | - | | was defined. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~object.__dict__`| The namespace supporting | Writable | - | | arbitrary function | | - | | attributes. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | - | | that contain bindings for the | | - | | function's free variables. | | - | | See below for information on | | - | | the ``cell_contents`` | | - | | attribute. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__annotations__` | A dict containing annotations | Writable | - | | of parameters. The keys of | | - | | the dict are the parameter | | - | | names, and ``'return'`` for | | - | | the return annotation, if | | - | | provided. For more | | - | | information on working with | | - | | this attribute, see | | - | | :ref:`annotations-howto`. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__kwdefaults__` | A dict containing defaults | Writable | - | | for keyword-only parameters. | | - +-------------------------+-------------------------------+-----------+ - - Most of the attributes labelled "Writable" check the type of the assigned value. - - Function objects also support getting and setting arbitrary attributes, which - can be used, for example, to attach metadata to functions. Regular attribute - dot-notation is used to get and set such attributes. *Note that the current - implementation only supports function attributes on user-defined functions. - Function attributes on built-in functions may be supported in the future.* - - A cell object has the attribute ``cell_contents``. This can be used to get - the value of the cell, as well as set the value. - - Additional information about a function's definition can be retrieved from its - code object; see the description of internal types below. The - :data:`cell ` type can be accessed in the :mod:`types` - module. - - Instance methods - .. index:: - pair: object; method - pair: object; user-defined method - pair: user-defined; method - - An instance method object combines a class, a class instance and any - callable object (normally a user-defined function). - - .. index:: - single: __func__ (method attribute) - single: __self__ (method attribute) - single: __doc__ (method attribute) - single: __name__ (method attribute) - single: __module__ (method attribute) - - Special read-only attributes: :attr:`__self__` is the class instance object, - :attr:`__func__` is the function object; :attr:`__doc__` is the method's - documentation (same as ``__func__.__doc__``); :attr:`~definition.__name__` is the - method name (same as ``__func__.__name__``); :attr:`__module__` is the - name of the module the method was defined in, or ``None`` if unavailable. - - Methods also support accessing (but not setting) the arbitrary function - attributes on the underlying function object. - - User-defined method objects may be created when getting an attribute of a - class (perhaps via an instance of that class), if that attribute is a - user-defined function object or a class method object. - - When an instance method object is created by retrieving a user-defined - function object from a class via one of its instances, its - :attr:`__self__` attribute is the instance, and the method object is said - to be bound. The new method's :attr:`__func__` attribute is the original - function object. - - When an instance method object is created by retrieving a class method - object from a class or instance, its :attr:`__self__` attribute is the - class itself, and its :attr:`__func__` attribute is the function object - underlying the class method. - - When an instance method object is called, the underlying function - (:attr:`__func__`) is called, inserting the class instance - (:attr:`__self__`) in front of the argument list. For instance, when - :class:`C` is a class which contains a definition for a function - :meth:`f`, and ``x`` is an instance of :class:`C`, calling ``x.f(1)`` is - equivalent to calling ``C.f(x, 1)``. - - When an instance method object is derived from a class method object, the - "class instance" stored in :attr:`__self__` will actually be the class - itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to - calling ``f(C,1)`` where ``f`` is the underlying function. - - Note that the transformation from function object to instance method - object happens each time the attribute is retrieved from the instance. In - some cases, a fruitful optimization is to assign the attribute to a local - variable and call that local variable. Also notice that this - transformation only happens for user-defined functions; other callable - objects (and all non-callable objects) are retrieved without - transformation. It is also important to note that user-defined functions - which are attributes of a class instance are not converted to bound - methods; this *only* happens when the function is an attribute of the - class. - - Generator functions - .. index:: - single: generator; function - single: generator; iterator - - A function or method which uses the :keyword:`yield` statement (see section - :ref:`yield`) is called a :dfn:`generator function`. Such a function, when - called, always returns an :term:`iterator` object which can be used to - execute the body of the function: calling the iterator's - :meth:`iterator.__next__` method will cause the function to execute until - it provides a value using the :keyword:`!yield` statement. When the - function executes a :keyword:`return` statement or falls off the end, a - :exc:`StopIteration` exception is raised and the iterator will have - reached the end of the set of values to be returned. - - Coroutine functions - .. index:: - single: coroutine; function - - A function or method which is defined using :keyword:`async def` is called - a :dfn:`coroutine function`. Such a function, when called, returns a - :term:`coroutine` object. It may contain :keyword:`await` expressions, - as well as :keyword:`async with` and :keyword:`async for` statements. See - also the :ref:`coroutine-objects` section. - - Asynchronous generator functions - .. index:: - single: asynchronous generator; function - single: asynchronous generator; asynchronous iterator - - A function or method which is defined using :keyword:`async def` and - which uses the :keyword:`yield` statement is called a - :dfn:`asynchronous generator function`. Such a function, when called, - returns an :term:`asynchronous iterator` object which can be used in an - :keyword:`async for` statement to execute the body of the function. - - Calling the asynchronous iterator's - :meth:`aiterator.__anext__ ` method - will return an :term:`awaitable` which when awaited - will execute until it provides a value using the :keyword:`yield` - expression. When the function executes an empty :keyword:`return` - statement or falls off the end, a :exc:`StopAsyncIteration` exception - is raised and the asynchronous iterator will have reached the end of - the set of values to be yielded. - - Built-in functions - .. index:: - pair: object; built-in function - pair: object; function - pair: C; language - - A built-in function object is a wrapper around a C function. Examples of - built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a - standard built-in module). The number and type of the arguments are - determined by the C function. Special read-only attributes: - :attr:`__doc__` is the function's documentation string, or ``None`` if - unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is - set to ``None`` (but see the next item); :attr:`__module__` is the name of - the module the function was defined in or ``None`` if unavailable. - - Built-in methods - .. index:: - pair: object; built-in method - pair: object; method - pair: built-in; method - - This is really a different disguise of a built-in function, this time containing - an object passed to the C function as an implicit extra argument. An example of - a built-in method is ``alist.append()``, assuming *alist* is a list object. In - this case, the special read-only attribute :attr:`__self__` is set to the object - denoted by *alist*. - - Classes - Classes are callable. These objects normally act as factories for new - instances of themselves, but variations are possible for class types that - override :meth:`~object.__new__`. The arguments of the call are passed to - :meth:`__new__` and, in the typical case, to :meth:`~object.__init__` to - initialize the new instance. - - Class Instances - Instances of arbitrary classes can be made callable by defining a - :meth:`~object.__call__` method in their class. +-------------- + +.. index:: + pair: object; callable + pair: function; call + single: invocation + pair: function; argument + +These are the types to which the function call operation (see section +:ref:`calls`) can be applied: + + +.. _user-defined-funcs: + +User-defined functions +^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: user-defined; function + pair: object; function + pair: object; user-defined function + +A user-defined function object is created by a function definition (see +section :ref:`function`). It should be called with an argument list +containing the same number of items as the function's formal parameter +list. + +Special attributes: + +.. tabularcolumns:: |l|L|l| + +.. index:: + single: __doc__ (function attribute) + single: __name__ (function attribute) + single: __module__ (function attribute) + single: __dict__ (function attribute) + single: __defaults__ (function attribute) + single: __closure__ (function attribute) + single: __code__ (function attribute) + single: __globals__ (function attribute) + single: __annotations__ (function attribute) + single: __kwdefaults__ (function attribute) + single: __type_params__ (function attribute) + pair: global; namespace + ++-------------------------+-------------------------------+-----------+ +| Attribute | Meaning | | ++=========================+===============================+===========+ +| :attr:`__doc__` | The function's documentation | Writable | +| | string, or ``None`` if | | +| | unavailable; not inherited by | | +| | subclasses. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`~definition.\ | The function's name. | Writable | +| __name__` | | | ++-------------------------+-------------------------------+-----------+ +| :attr:`~definition.\ | The function's | Writable | +| __qualname__` | :term:`qualified name`. | | +| | | | +| | .. versionadded:: 3.3 | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__module__` | The name of the module the | Writable | +| | function was defined in, or | | +| | ``None`` if unavailable. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__defaults__` | A tuple containing default | Writable | +| | argument values for those | | +| | arguments that have defaults, | | +| | or ``None`` if no arguments | | +| | have a default value. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__code__` | The code object representing | Writable | +| | the compiled function body. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__globals__` | A reference to the dictionary | Read-only | +| | that holds the function's | | +| | global variables --- the | | +| | global namespace of the | | +| | module in which the function | | +| | was defined. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`~object.__dict__`| The namespace supporting | Writable | +| | arbitrary function | | +| | attributes. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | +| | that contain bindings for the | | +| | function's free variables. | | +| | See below for information on | | +| | the ``cell_contents`` | | +| | attribute. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__annotations__` | A dict containing annotations | Writable | +| | of parameters. The keys of | | +| | the dict are the parameter | | +| | names, and ``'return'`` for | | +| | the return annotation, if | | +| | provided. For more | | +| | information on working with | | +| | this attribute, see | | +| | :ref:`annotations-howto`. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__kwdefaults__` | A dict containing defaults | Writable | +| | for keyword-only parameters. | | ++-------------------------+-------------------------------+-----------+ +| :attr:`__type_params__` | A tuple containing the | Writable | +| | :ref:`type parameters | | +| | ` of a | | +| | :ref:`generic function | | +| | `. | | ++-------------------------+-------------------------------+-----------+ + +Most of the attributes labelled "Writable" check the type of the assigned value. + +Function objects also support getting and setting arbitrary attributes, which +can be used, for example, to attach metadata to functions. Regular attribute +dot-notation is used to get and set such attributes. *Note that the current +implementation only supports function attributes on user-defined functions. +Function attributes on built-in functions may be supported in the future.* + +A cell object has the attribute ``cell_contents``. This can be used to get +the value of the cell, as well as set the value. + +Additional information about a function's definition can be retrieved from its +code object; see the description of internal types below. The +:data:`cell ` type can be accessed in the :mod:`types` +module. + + +.. _instance-methods: + +Instance methods +^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; method + pair: object; user-defined method + pair: user-defined; method + +An instance method object combines a class, a class instance and any +callable object (normally a user-defined function). + +.. index:: + single: __func__ (method attribute) + single: __self__ (method attribute) + single: __doc__ (method attribute) + single: __name__ (method attribute) + single: __module__ (method attribute) + +Special read-only attributes: + +.. list-table:: + + * - .. attribute:: method.__self__ + - Refers to the class instance object to which the method is + :ref:`bound ` + + * - .. attribute:: method.__func__ + - Refers to the original function object + + * - .. attribute:: method.__doc__ + - The method's documentation (same as :attr:`!method.__func__.__doc__`). + A :class:`string ` if the original function had a docstring, else + ``None``. + + * - .. attribute:: method.__name__ + - The name of the method (same as :attr:`!method.__func__.__name__`) + + * - .. attribute:: method.__module__ + - The name of the module the method was defined in, or ``None`` if + unavailable. + +Methods also support accessing (but not setting) the arbitrary function +attributes on the underlying :ref:`function object `. + +User-defined method objects may be created when getting an attribute of a +class (perhaps via an instance of that class), if that attribute is a +user-defined :ref:`function object ` or a +:class:`classmethod` object. + +.. _method-binding: + +When an instance method object is created by retrieving a user-defined +:ref:`function object ` from a class via one of its +instances, its :attr:`~method.__self__` attribute is the instance, and the +method object is said to be *bound*. The new method's :attr:`~method.__func__` +attribute is the original function object. + +When an instance method object is created by retrieving a :class:`classmethod` +object from a class or instance, its :attr:`~method.__self__` attribute is the +class itself, and its :attr:`~method.__func__` attribute is the function object +underlying the class method. + +When an instance method object is called, the underlying function +(:attr:`~method.__func__`) is called, inserting the class instance +(:attr:`~method.__self__`) in front of the argument list. For instance, when +:class:`!C` is a class which contains a definition for a function +:meth:`!f`, and ``x`` is an instance of :class:`!C`, calling ``x.f(1)`` is +equivalent to calling ``C.f(x, 1)``. + +When an instance method object is derived from a :class:`classmethod` object, the +"class instance" stored in :attr:`~method.__self__` will actually be the class +itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to +calling ``f(C,1)`` where ``f`` is the underlying function. + +Note that the transformation from :ref:`function object ` +to instance method +object happens each time the attribute is retrieved from the instance. In +some cases, a fruitful optimization is to assign the attribute to a local +variable and call that local variable. Also notice that this +transformation only happens for user-defined functions; other callable +objects (and all non-callable objects) are retrieved without +transformation. It is also important to note that user-defined functions +which are attributes of a class instance are not converted to bound +methods; this *only* happens when the function is an attribute of the +class. + + +Generator functions +^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: generator; function + single: generator; iterator + +A function or method which uses the :keyword:`yield` statement (see section +:ref:`yield`) is called a :dfn:`generator function`. Such a function, when +called, always returns an :term:`iterator` object which can be used to +execute the body of the function: calling the iterator's +:meth:`iterator.__next__` method will cause the function to execute until +it provides a value using the :keyword:`!yield` statement. When the +function executes a :keyword:`return` statement or falls off the end, a +:exc:`StopIteration` exception is raised and the iterator will have +reached the end of the set of values to be returned. + + +Coroutine functions +^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: coroutine; function + +A function or method which is defined using :keyword:`async def` is called +a :dfn:`coroutine function`. Such a function, when called, returns a +:term:`coroutine` object. It may contain :keyword:`await` expressions, +as well as :keyword:`async with` and :keyword:`async for` statements. See +also the :ref:`coroutine-objects` section. + + +Asynchronous generator functions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: asynchronous generator; function + single: asynchronous generator; asynchronous iterator + +A function or method which is defined using :keyword:`async def` and +which uses the :keyword:`yield` statement is called a +:dfn:`asynchronous generator function`. Such a function, when called, +returns an :term:`asynchronous iterator` object which can be used in an +:keyword:`async for` statement to execute the body of the function. + +Calling the asynchronous iterator's +:meth:`aiterator.__anext__ ` method +will return an :term:`awaitable` which when awaited +will execute until it provides a value using the :keyword:`yield` +expression. When the function executes an empty :keyword:`return` +statement or falls off the end, a :exc:`StopAsyncIteration` exception +is raised and the asynchronous iterator will have reached the end of +the set of values to be yielded. + + +Built-in functions +^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; built-in function + pair: object; function + pair: C; language + +A built-in function object is a wrapper around a C function. Examples of +built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a +standard built-in module). The number and type of the arguments are +determined by the C function. Special read-only attributes: +:attr:`__doc__` is the function's documentation string, or ``None`` if +unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is +set to ``None`` (but see the next item); :attr:`__module__` is the name of +the module the function was defined in or ``None`` if unavailable. + + +.. _builtin-methods: + +Built-in methods +^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; built-in method + pair: object; method + pair: built-in; method + +This is really a different disguise of a built-in function, this time containing +an object passed to the C function as an implicit extra argument. An example of +a built-in method is ``alist.append()``, assuming *alist* is a list object. In +this case, the special read-only attribute :attr:`!__self__` is set to the object +denoted by *alist*. (The attribute has the same semantics as it does with +:attr:`other instance methods `.) + + +Classes +^^^^^^^ + +Classes are callable. These objects normally act as factories for new +instances of themselves, but variations are possible for class types that +override :meth:`~object.__new__`. The arguments of the call are passed to +:meth:`!__new__` and, in the typical case, to :meth:`~object.__init__` to +initialize the new instance. + + +Class Instances +^^^^^^^^^^^^^^^ + +Instances of arbitrary classes can be made callable by defining a +:meth:`~object.__call__` method in their class. Modules - .. index:: - pair: statement; import - pair: object; module - - Modules are a basic organizational unit of Python code, and are created by - the :ref:`import system ` as invoked either by the - :keyword:`import` statement, or by calling - functions such as :func:`importlib.import_module` and built-in - :func:`__import__`. A module object has a namespace implemented by a - dictionary object (this is the dictionary referenced by the ``__globals__`` - attribute of functions defined in the module). Attribute references are - translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to - ``m.__dict__["x"]``. A module object does not contain the code object used - to initialize the module (since it isn't needed once the initialization is - done). - - Attribute assignment updates the module's namespace dictionary, e.g., - ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``. +------- - .. index:: - single: __name__ (module attribute) - single: __doc__ (module attribute) - single: __file__ (module attribute) - single: __annotations__ (module attribute) - pair: module; namespace +.. index:: + pair: statement; import + pair: object; module + +Modules are a basic organizational unit of Python code, and are created by +the :ref:`import system ` as invoked either by the +:keyword:`import` statement, or by calling +functions such as :func:`importlib.import_module` and built-in +:func:`__import__`. A module object has a namespace implemented by a +dictionary object (this is the dictionary referenced by the ``__globals__`` +attribute of functions defined in the module). Attribute references are +translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to +``m.__dict__["x"]``. A module object does not contain the code object used +to initialize the module (since it isn't needed once the initialization is +done). + +Attribute assignment updates the module's namespace dictionary, e.g., +``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``. - Predefined (writable) attributes: +.. index:: + single: __name__ (module attribute) + single: __doc__ (module attribute) + single: __file__ (module attribute) + single: __annotations__ (module attribute) + pair: module; namespace - :attr:`__name__` - The module's name. +Predefined (writable) attributes: - :attr:`__doc__` - The module's documentation string, or ``None`` if - unavailable. + :attr:`__name__` + The module's name. - :attr:`__file__` - The pathname of the file from which the - module was loaded, if it was loaded from a file. - The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter. For extension modules - loaded dynamically from a shared library, it's the pathname of the shared - library file. + :attr:`__doc__` + The module's documentation string, or ``None`` if + unavailable. - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` collected during - module body execution. For best practices on working - with :attr:`__annotations__`, please see :ref:`annotations-howto`. + :attr:`__file__` + The pathname of the file from which the + module was loaded, if it was loaded from a file. + The :attr:`__file__` + attribute may be missing for certain types of modules, such as C modules + that are statically linked into the interpreter. For extension modules + loaded dynamically from a shared library, it's the pathname of the shared + library file. - .. index:: single: __dict__ (module attribute) + :attr:`__annotations__` + A dictionary containing + :term:`variable annotations ` collected during + module body execution. For best practices on working + with :attr:`__annotations__`, please see :ref:`annotations-howto`. - Special read-only attribute: :attr:`~object.__dict__` is the module's - namespace as a dictionary object. +.. index:: single: __dict__ (module attribute) - .. impl-detail:: +Special read-only attribute: :attr:`~object.__dict__` is the module's +namespace as a dictionary object. + +.. impl-detail:: + + Because of the way CPython clears module dictionaries, the module + dictionary will be cleared when the module falls out of scope even if the + dictionary still has live references. To avoid this, copy the dictionary + or keep the module around while using its dictionary directly. - Because of the way CPython clears module dictionaries, the module - dictionary will be cleared when the module falls out of scope even if the - dictionary still has live references. To avoid this, copy the dictionary - or keep the module around while using its dictionary directly. Custom classes - Custom class types are typically created by class definitions (see section - :ref:`class`). A class has a namespace implemented by a dictionary object. - Class attribute references are translated to lookups in this dictionary, e.g., - ``C.x`` is translated to ``C.__dict__["x"]`` (although there are a number of - hooks which allow for other means of locating attributes). When the attribute - name is not found there, the attribute search continues in the base classes. - This search of the base classes uses the C3 method resolution order which - behaves correctly even in the presence of 'diamond' inheritance structures - where there are multiple inheritance paths leading back to a common ancestor. - Additional details on the C3 MRO used by Python can be found in the - documentation accompanying the 2.3 release at - https://www.python.org/download/releases/2.3/mro/. - - .. XXX: Could we add that MRO doc as an appendix to the language ref? +-------------- + +Custom class types are typically created by class definitions (see section +:ref:`class`). A class has a namespace implemented by a dictionary object. +Class attribute references are translated to lookups in this dictionary, e.g., +``C.x`` is translated to ``C.__dict__["x"]`` (although there are a number of +hooks which allow for other means of locating attributes). When the attribute +name is not found there, the attribute search continues in the base classes. +This search of the base classes uses the C3 method resolution order which +behaves correctly even in the presence of 'diamond' inheritance structures +where there are multiple inheritance paths leading back to a common ancestor. +Additional details on the C3 MRO used by Python can be found in the +documentation accompanying the 2.3 release at +https://www.python.org/download/releases/2.3/mro/. + +.. XXX: Could we add that MRO doc as an appendix to the language ref? - .. index:: - pair: object; class - pair: object; class instance - pair: object; instance - pair: class object; call - single: container - pair: object; dictionary - pair: class; attribute +.. index:: + pair: object; class + pair: object; class instance + pair: object; instance + pair: class object; call + single: container + pair: object; dictionary + pair: class; attribute - When a class attribute reference (for class :class:`C`, say) would yield a - class method object, it is transformed into an instance method object whose - :attr:`__self__` attribute is :class:`C`. When it would yield a static - method object, it is transformed into the object wrapped by the static method - object. See section :ref:`descriptors` for another way in which attributes - retrieved from a class may differ from those actually contained in its - :attr:`~object.__dict__`. +When a class attribute reference (for class :class:`!C`, say) would yield a +class method object, it is transformed into an instance method object whose +:attr:`~method.__self__` attribute is :class:`!C`. +When it would yield a :class:`staticmethod` object, +it is transformed into the object wrapped by the static method +object. See section :ref:`descriptors` for another way in which attributes +retrieved from a class may differ from those actually contained in its +:attr:`~object.__dict__`. - .. index:: triple: class; attribute; assignment +.. index:: triple: class; attribute; assignment - Class attribute assignments update the class's dictionary, never the dictionary - of a base class. +Class attribute assignments update the class's dictionary, never the dictionary +of a base class. - .. index:: pair: class object; call +.. index:: pair: class object; call - A class object can be called (see above) to yield a class instance (see below). +A class object can be called (see above) to yield a class instance (see below). - .. index:: - single: __name__ (class attribute) - single: __module__ (class attribute) - single: __dict__ (class attribute) - single: __bases__ (class attribute) - single: __doc__ (class attribute) - single: __annotations__ (class attribute) +.. index:: + single: __name__ (class attribute) + single: __module__ (class attribute) + single: __dict__ (class attribute) + single: __bases__ (class attribute) + single: __doc__ (class attribute) + single: __annotations__ (class attribute) + single: __type_params__ (class attribute) - Special attributes: +Special attributes: - :attr:`~definition.__name__` - The class name. + :attr:`~definition.__name__` + The class name. - :attr:`__module__` - The name of the module in which the class was defined. + :attr:`__module__` + The name of the module in which the class was defined. - :attr:`~object.__dict__` - The dictionary containing the class's namespace. + :attr:`~object.__dict__` + The dictionary containing the class's namespace. + + :attr:`~class.__bases__` + A tuple containing the base classes, in the order of + their occurrence in the base class list. - :attr:`~class.__bases__` - A tuple containing the base classes, in the order of - their occurrence in the base class list. + :attr:`__doc__` + The class's documentation string, or ``None`` if undefined. - :attr:`__doc__` - The class's documentation string, or ``None`` if undefined. + :attr:`__annotations__` + A dictionary containing + :term:`variable annotations ` + collected during class body execution. For best practices on + working with :attr:`__annotations__`, please see + :ref:`annotations-howto`. + + :attr:`__type_params__` + A tuple containing the :ref:`type parameters ` of + a :ref:`generic class `. - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` - collected during class body execution. For best practices on - working with :attr:`__annotations__`, please see - :ref:`annotations-howto`. Class instances - .. index:: - pair: object; class instance - pair: object; instance - pair: class; instance - pair: class instance; attribute - - A class instance is created by calling a class object (see above). A class - instance has a namespace implemented as a dictionary which is the first place - in which attribute references are searched. When an attribute is not found - there, and the instance's class has an attribute by that name, the search - continues with the class attributes. If a class attribute is found that is a - user-defined function object, it is transformed into an instance method - object whose :attr:`__self__` attribute is the instance. Static method and - class method objects are also transformed; see above under "Classes". See - section :ref:`descriptors` for another way in which attributes of a class - retrieved via its instances may differ from the objects actually stored in - the class's :attr:`~object.__dict__`. If no class attribute is found, and the - object's class has a :meth:`~object.__getattr__` method, that is called to satisfy - the lookup. - - .. index:: triple: class instance; attribute; assignment - - Attribute assignments and deletions update the instance's dictionary, never a - class's dictionary. If the class has a :meth:`~object.__setattr__` or - :meth:`~object.__delattr__` method, this is called instead of updating the instance - dictionary directly. +--------------- - .. index:: - pair: object; numeric - pair: object; sequence - pair: object; mapping +.. index:: + pair: object; class instance + pair: object; instance + pair: class; instance + pair: class instance; attribute + +A class instance is created by calling a class object (see above). A class +instance has a namespace implemented as a dictionary which is the first place +in which attribute references are searched. When an attribute is not found +there, and the instance's class has an attribute by that name, the search +continues with the class attributes. If a class attribute is found that is a +user-defined function object, it is transformed into an instance method +object whose :attr:`~method.__self__` attribute is the instance. Static method and +class method objects are also transformed; see above under "Classes". See +section :ref:`descriptors` for another way in which attributes of a class +retrieved via its instances may differ from the objects actually stored in +the class's :attr:`~object.__dict__`. If no class attribute is found, and the +object's class has a :meth:`~object.__getattr__` method, that is called to satisfy +the lookup. + +.. index:: triple: class instance; attribute; assignment + +Attribute assignments and deletions update the instance's dictionary, never a +class's dictionary. If the class has a :meth:`~object.__setattr__` or +:meth:`~object.__delattr__` method, this is called instead of updating the instance +dictionary directly. - Class instances can pretend to be numbers, sequences, or mappings if they have - methods with certain special names. See section :ref:`specialnames`. +.. index:: + pair: object; numeric + pair: object; sequence + pair: object; mapping - .. index:: - single: __dict__ (instance attribute) - single: __class__ (instance attribute) +Class instances can pretend to be numbers, sequences, or mappings if they have +methods with certain special names. See section :ref:`specialnames`. + +.. index:: + single: __dict__ (instance attribute) + single: __class__ (instance attribute) + +Special attributes: :attr:`~object.__dict__` is the attribute dictionary; +:attr:`~instance.__class__` is the instance's class. - Special attributes: :attr:`~object.__dict__` is the attribute dictionary; - :attr:`~instance.__class__` is the instance's class. I/O objects (also known as file objects) - .. index:: - pair: built-in function; open - pair: module; io - single: popen() (in module os) - single: makefile() (socket method) - single: sys.stdin - single: sys.stdout - single: sys.stderr - single: stdio - single: stdin (in module sys) - single: stdout (in module sys) - single: stderr (in module sys) - - A :term:`file object` represents an open file. Various shortcuts are - available to create file objects: the :func:`open` built-in function, and - also :func:`os.popen`, :func:`os.fdopen`, and the - :meth:`~socket.socket.makefile` method of socket objects (and perhaps by - other functions or methods provided by extension modules). - - The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are - initialized to file objects corresponding to the interpreter's standard - input, output and error streams; they are all open in text mode and - therefore follow the interface defined by the :class:`io.TextIOBase` - abstract class. +---------------------------------------- + +.. index:: + pair: built-in function; open + pair: module; io + single: popen() (in module os) + single: makefile() (socket method) + single: sys.stdin + single: sys.stdout + single: sys.stderr + single: stdio + single: stdin (in module sys) + single: stdout (in module sys) + single: stderr (in module sys) + +A :term:`file object` represents an open file. Various shortcuts are +available to create file objects: the :func:`open` built-in function, and +also :func:`os.popen`, :func:`os.fdopen`, and the +:meth:`~socket.socket.makefile` method of socket objects (and perhaps by +other functions or methods provided by extension modules). + +The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are +initialized to file objects corresponding to the interpreter's standard +input, output and error streams; they are all open in text mode and +therefore follow the interface defined by the :class:`io.TextIOBase` +abstract class. + Internal types - .. index:: - single: internal type - single: types, internal - - A few types used internally by the interpreter are exposed to the user. Their - definitions may change with future versions of the interpreter, but they are - mentioned here for completeness. - - .. index:: bytecode, object; code, code object - - Code objects - Code objects represent *byte-compiled* executable Python code, or :term:`bytecode`. - The difference between a code object and a function object is that the function - object contains an explicit reference to the function's globals (the module in - which it was defined), while a code object contains no context; also the default - argument values are stored in the function object, not in the code object - (because they represent values calculated at run-time). Unlike function - objects, code objects are immutable and contain no references (directly or - indirectly) to mutable objects. - - .. index:: - single: co_argcount (code object attribute) - single: co_posonlyargcount (code object attribute) - single: co_kwonlyargcount (code object attribute) - single: co_code (code object attribute) - single: co_consts (code object attribute) - single: co_filename (code object attribute) - single: co_firstlineno (code object attribute) - single: co_flags (code object attribute) - single: co_lnotab (code object attribute) - single: co_name (code object attribute) - single: co_names (code object attribute) - single: co_nlocals (code object attribute) - single: co_stacksize (code object attribute) - single: co_varnames (code object attribute) - single: co_cellvars (code object attribute) - single: co_freevars (code object attribute) - single: co_qualname (code object attribute) - - Special read-only attributes: :attr:`co_name` gives the function name; - :attr:`co_qualname` gives the fully qualified function name; - :attr:`co_argcount` is the total number of positional arguments - (including positional-only arguments and arguments with default values); - :attr:`co_posonlyargcount` is the number of positional-only arguments - (including arguments with default values); :attr:`co_kwonlyargcount` is - the number of keyword-only arguments (including arguments with default - values); :attr:`co_nlocals` is the number of local variables used by the - function (including arguments); :attr:`co_varnames` is a tuple containing - the names of the local variables (starting with the argument names); - :attr:`co_cellvars` is a tuple containing the names of local variables - that are referenced by nested functions; :attr:`co_freevars` is a tuple - containing the names of free variables; :attr:`co_code` is a string - representing the sequence of bytecode instructions; :attr:`co_consts` is - a tuple containing the literals used by the bytecode; :attr:`co_names` is - a tuple containing the names used by the bytecode; :attr:`co_filename` is - the filename from which the code was compiled; :attr:`co_firstlineno` is - the first line number of the function; :attr:`co_lnotab` is a string - encoding the mapping from bytecode offsets to line numbers (for details - see the source code of the interpreter, is deprecated since 3.12 - and may be removed in 3.14); :attr:`co_stacksize` is the - required stack size; :attr:`co_flags` is an integer encoding a number - of flags for the interpreter. - - .. index:: pair: object; generator - - The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if - the function uses the ``*arguments`` syntax to accept an arbitrary number of - positional arguments; bit ``0x08`` is set if the function uses the - ``**keywords`` syntax to accept arbitrary keyword arguments; bit ``0x20`` is set - if the function is a generator. - - Future feature declarations (``from __future__ import division``) also use bits - in :attr:`co_flags` to indicate whether a code object was compiled with a - particular feature enabled: bit ``0x2000`` is set if the function was compiled - with future division enabled; bits ``0x10`` and ``0x1000`` were used in earlier - versions of Python. - - Other bits in :attr:`co_flags` are reserved for internal use. - - .. index:: single: documentation string - - If a code object represents a function, the first item in :attr:`co_consts` is - the documentation string of the function, or ``None`` if undefined. - - .. method:: codeobject.co_positions() - - Returns an iterable over the source code positions of each bytecode - instruction in the code object. - - The iterator returns tuples containing the ``(start_line, end_line, - start_column, end_column)``. The *i-th* tuple corresponds to the - position of the source code that compiled to the *i-th* instruction. - Column information is 0-indexed utf-8 byte offsets on the given source - line. - - This positional information can be missing. A non-exhaustive lists of - cases where this may happen: - - - Running the interpreter with :option:`-X` ``no_debug_ranges``. - - Loading a pyc file compiled while using :option:`-X` ``no_debug_ranges``. - - Position tuples corresponding to artificial instructions. - - Line and column numbers that can't be represented due to - implementation specific limitations. - - When this occurs, some or all of the tuple elements can be - :const:`None`. - - .. versionadded:: 3.11 - - .. note:: - This feature requires storing column positions in code objects which may - result in a small increase of disk usage of compiled Python files or - interpreter memory usage. To avoid storing the extra information and/or - deactivate printing the extra traceback information, the - :option:`-X` ``no_debug_ranges`` command line flag or the :envvar:`PYTHONNODEBUGRANGES` - environment variable can be used. - - .. _frame-objects: - - Frame objects - .. index:: pair: object; frame - - Frame objects represent execution frames. They may occur in traceback objects - (see below), and are also passed to registered trace functions. - - .. index:: - single: f_back (frame attribute) - single: f_code (frame attribute) - single: f_globals (frame attribute) - single: f_locals (frame attribute) - single: f_lasti (frame attribute) - single: f_builtins (frame attribute) - - Special read-only attributes: :attr:`f_back` is to the previous stack frame - (towards the caller), or ``None`` if this is the bottom stack frame; - :attr:`f_code` is the code object being executed in this frame; :attr:`f_locals` - is the dictionary used to look up local variables; :attr:`f_globals` is used for - global variables; :attr:`f_builtins` is used for built-in (intrinsic) names; - :attr:`f_lasti` gives the precise instruction (this is an index into the - bytecode string of the code object). - - Accessing ``f_code`` raises an :ref:`auditing event ` - ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. - - .. index:: - single: f_trace (frame attribute) - single: f_trace_lines (frame attribute) - single: f_trace_opcodes (frame attribute) - single: f_lineno (frame attribute) - - Special writable attributes: :attr:`f_trace`, if not ``None``, is a function - called for various events during code execution (this is used by the debugger). - Normally an event is triggered for each new source line - this can be - disabled by setting :attr:`f_trace_lines` to :const:`False`. - - Implementations *may* allow per-opcode events to be requested by setting - :attr:`f_trace_opcodes` to :const:`True`. Note that this may lead to - undefined interpreter behaviour if exceptions raised by the trace - function escape to the function being traced. - - :attr:`f_lineno` is the current line number of the frame --- writing to this - from within a trace function jumps to the given line (only for the bottom-most - frame). A debugger can implement a Jump command (aka Set Next Statement) - by writing to f_lineno. - - Frame objects support one method: - - .. method:: frame.clear() - - This method clears all references to local variables held by the - frame. Also, if the frame belonged to a generator, the generator - is finalized. This helps break reference cycles involving frame - objects (for example when catching an exception and storing its - traceback for later use). - - :exc:`RuntimeError` is raised if the frame is currently executing. - - .. versionadded:: 3.4 - - .. _traceback-objects: - - Traceback objects - .. index:: - pair: object; traceback - pair: stack; trace - pair: exception; handler - pair: execution; stack - single: exc_info (in module sys) - single: last_traceback (in module sys) - single: sys.exc_info - single: sys.exception - single: sys.last_traceback - - Traceback objects represent a stack trace of an exception. A traceback object - is implicitly created when an exception occurs, and may also be explicitly - created by calling :class:`types.TracebackType`. - - For implicitly created tracebacks, when the search for an exception handler - unwinds the execution stack, at each unwound level a traceback object is - inserted in front of the current traceback. When an exception handler is - entered, the stack trace is made available to the program. (See section - :ref:`try`.) It is accessible as the third item of the - tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute - of the caught exception. - - When the program contains no suitable - handler, the stack trace is written (nicely formatted) to the standard error - stream; if the interpreter is interactive, it is also made available to the user - as ``sys.last_traceback``. - - For explicitly created tracebacks, it is up to the creator of the traceback - to determine how the ``tb_next`` attributes should be linked to form a - full stack trace. - - .. index:: - single: tb_frame (traceback attribute) - single: tb_lineno (traceback attribute) - single: tb_lasti (traceback attribute) - pair: statement; try - - Special read-only attributes: - :attr:`tb_frame` points to the execution frame of the current level; - :attr:`tb_lineno` gives the line number where the exception occurred; - :attr:`tb_lasti` indicates the precise instruction. - The line number and last instruction in the traceback may differ from the - line number of its frame object if the exception occurred in a - :keyword:`try` statement with no matching except clause or with a - finally clause. - - Accessing ``tb_frame`` raises an :ref:`auditing event ` - ``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. - - .. index:: - single: tb_next (traceback attribute) - - Special writable attribute: :attr:`tb_next` is the next level in the stack - trace (towards the frame where the exception occurred), or ``None`` if - there is no next level. - - .. versionchanged:: 3.7 - Traceback objects can now be explicitly instantiated from Python code, - and the ``tb_next`` attribute of existing instances can be updated. - - Slice objects - .. index:: pair: built-in function; slice - - Slice objects are used to represent slices for - :meth:`~object.__getitem__` - methods. They are also created by the built-in :func:`slice` function. - - .. index:: - single: start (slice object attribute) - single: stop (slice object attribute) - single: step (slice object attribute) - - Special read-only attributes: :attr:`~slice.start` is the lower bound; - :attr:`~slice.stop` is the upper bound; :attr:`~slice.step` is the step - value; each is ``None`` if omitted. These attributes can have any type. - - Slice objects support one method: - - .. method:: slice.indices(self, length) - - This method takes a single integer argument *length* and computes - information about the slice that the slice object would describe if - applied to a sequence of *length* items. It returns a tuple of three - integers; respectively these are the *start* and *stop* indices and the - *step* or stride length of the slice. Missing or out-of-bounds indices - are handled in a manner consistent with regular slices. - - Static method objects - Static method objects provide a way of defeating the transformation of function - objects to method objects described above. A static method object is a wrapper - around any other object, usually a user-defined method object. When a static - method object is retrieved from a class or a class instance, the object actually - returned is the wrapped object, which is not subject to any further - transformation. Static method objects are also callable. Static method - objects are created by the built-in :func:`staticmethod` constructor. - - Class method objects - A class method object, like a static method object, is a wrapper around another - object that alters the way in which that object is retrieved from classes and - class instances. The behaviour of class method objects upon such retrieval is - described above, under "User-defined methods". Class method objects are created - by the built-in :func:`classmethod` constructor. +-------------- + +.. index:: + single: internal type + single: types, internal + +A few types used internally by the interpreter are exposed to the user. Their +definitions may change with future versions of the interpreter, but they are +mentioned here for completeness. + + +.. _code-objects: + +Code objects +^^^^^^^^^^^^ + +.. index:: bytecode, object; code, code object + +Code objects represent *byte-compiled* executable Python code, or :term:`bytecode`. +The difference between a code object and a function object is that the function +object contains an explicit reference to the function's globals (the module in +which it was defined), while a code object contains no context; also the default +argument values are stored in the function object, not in the code object +(because they represent values calculated at run-time). Unlike function +objects, code objects are immutable and contain no references (directly or +indirectly) to mutable objects. + +.. index:: + single: co_argcount (code object attribute) + single: co_posonlyargcount (code object attribute) + single: co_kwonlyargcount (code object attribute) + single: co_code (code object attribute) + single: co_consts (code object attribute) + single: co_filename (code object attribute) + single: co_firstlineno (code object attribute) + single: co_flags (code object attribute) + single: co_lnotab (code object attribute) + single: co_name (code object attribute) + single: co_names (code object attribute) + single: co_nlocals (code object attribute) + single: co_stacksize (code object attribute) + single: co_varnames (code object attribute) + single: co_cellvars (code object attribute) + single: co_freevars (code object attribute) + single: co_qualname (code object attribute) + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + + * - .. attribute:: codeobject.co_name + - The function name + + * - .. attribute:: codeobject.co_qualname + - The fully qualified function name + + * - .. attribute:: codeobject.co_argcount + - The total number of positional :term:`parameters ` + (including positional-only parameters and parameters with default values) + that the function has + + * - .. attribute:: codeobject.co_posonlyargcount + - The number of positional-only :term:`parameters ` + (including arguments with default values) that the function has + + * - .. attribute:: codeobject.co_kwonlyargcount + - The number of keyword-only :term:`parameters ` + (including arguments with default values) that the function has + + * - .. attribute:: codeobject.co_nlocals + - The number of :ref:`local variables ` used by the function + (including parameters) + + * - .. attribute:: codeobject.co_varnames + - A :class:`tuple` containing the names of the local variables in the + function (starting with the parameter names) + + * - .. attribute:: codeobject.co_cellvars + - A :class:`tuple` containing the names of :ref:`local variables ` + that are referenced by nested functions inside the function + + * - .. attribute:: codeobject.co_freevars + - A :class:`tuple` containing the names of free variables in the function + + * - .. attribute:: codeobject.co_code + - A string representing the sequence of :term:`bytecode` instructions in + the function + + * - .. attribute:: codeobject.co_consts + - A :class:`tuple` containing the literals used by the :term:`bytecode` in + the function + + * - .. attribute:: codeobject.co_names + - A :class:`tuple` containing the names used by the :term:`bytecode` in + the function + + * - .. attribute:: codeobject.co_filename + - The name of the file from which the code was compiled + + * - .. attribute:: codeobject.co_firstlineno + - The line number of the first line of the function + + * - .. attribute:: codeobject.co_lnotab + - A string encoding the mapping from :term:`bytecode` offsets to line + numbers. For details, see the source code of the interpreter. + + .. deprecated:: 3.12 + This attribute of code objects is deprecated, and may be removed in + Python 3.14. + + * - .. attribute:: codeobject.co_stacksize + - The required stack size of the code object + + * - .. attribute:: codeobject.co_flags + - An :class:`integer ` encoding a number of flags for the + interpreter. + +.. index:: pair: object; generator + +The following flag bits are defined for :attr:`~codeobject.co_flags`: +bit ``0x04`` is set if +the function uses the ``*arguments`` syntax to accept an arbitrary number of +positional arguments; bit ``0x08`` is set if the function uses the +``**keywords`` syntax to accept arbitrary keyword arguments; bit ``0x20`` is set +if the function is a generator. See :ref:`inspect-module-co-flags` for details +on the semantics of each flags that might be present. + +Future feature declarations (``from __future__ import division``) also use bits +in :attr:`~codeobject.co_flags` to indicate whether a code object was compiled with a +particular feature enabled: bit ``0x2000`` is set if the function was compiled +with future division enabled; bits ``0x10`` and ``0x1000`` were used in earlier +versions of Python. + +Other bits in :attr:`~codeobject.co_flags` are reserved for internal use. + +.. index:: single: documentation string + +If a code object represents a function, the first item in +:attr:`~codeobject.co_consts` is +the documentation string of the function, or ``None`` if undefined. + +The :meth:`!co_positions` method +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. method:: codeobject.co_positions() + + Returns an iterable over the source code positions of each :term:`bytecode` + instruction in the code object. + + The iterator returns :class:`tuple`\s containing the ``(start_line, end_line, + start_column, end_column)``. The *i-th* tuple corresponds to the + position of the source code that compiled to the *i-th* instruction. + Column information is 0-indexed utf-8 byte offsets on the given source + line. + + This positional information can be missing. A non-exhaustive lists of + cases where this may happen: + + - Running the interpreter with :option:`-X` ``no_debug_ranges``. + - Loading a pyc file compiled while using :option:`-X` ``no_debug_ranges``. + - Position tuples corresponding to artificial instructions. + - Line and column numbers that can't be represented due to + implementation specific limitations. + + When this occurs, some or all of the tuple elements can be + :const:`None`. + + .. versionadded:: 3.11 + + .. note:: + This feature requires storing column positions in code objects which may + result in a small increase of disk usage of compiled Python files or + interpreter memory usage. To avoid storing the extra information and/or + deactivate printing the extra traceback information, the + :option:`-X` ``no_debug_ranges`` command line flag or the :envvar:`PYTHONNODEBUGRANGES` + environment variable can be used. + + +.. _frame-objects: + +Frame objects +^^^^^^^^^^^^^ + +.. index:: pair: object; frame + +Frame objects represent execution frames. They may occur in traceback objects +(see below), and are also passed to registered trace functions. + +.. index:: + single: f_back (frame attribute) + single: f_code (frame attribute) + single: f_globals (frame attribute) + single: f_locals (frame attribute) + single: f_lasti (frame attribute) + single: f_builtins (frame attribute) + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + + * - .. attribute:: frame.f_back + - Points to the previous stack frame (towards the caller), + or ``None`` if this is the bottom stack frame + + * - .. attribute:: frame.f_code + - The :ref:`code object ` being executed in this frame. + Accessing this attribute raises an :ref:`auditing event ` + ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. + + * - .. attribute:: frame.f_locals + - The dictionary used by the frame to look up + :ref:`local variables ` + + * - .. attribute:: frame.f_globals + - The dictionary used by the frame to look up + :ref:`global variables ` + + * - .. attribute:: frame.f_builtins + - The dictionary used by the frame to look up + :ref:`built-in (intrinsic) names ` + + * - .. attribute:: frame.f_lasti + - The "precise instruction" of the frame object + (this is an index into the :term:`bytecode` string of the + :ref:`code object `) + +.. index:: + single: f_trace (frame attribute) + single: f_trace_lines (frame attribute) + single: f_trace_opcodes (frame attribute) + single: f_lineno (frame attribute) + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + + * - .. attribute:: frame.f_trace + - If not ``None``, this is a function called for various events during + code execution (this is used by debuggers). Normally an event is + triggered for each new source line (see :attr:`~frame.f_trace_lines`). + + * - .. attribute:: frame.f_trace_lines + - Set this attribute to :const:`False` to disable triggering a tracing + event for each source line. + + * - .. attribute:: frame.f_trace_opcodes + - Set this attribute to :const:`True` to allow per-opcode events to be + requested. Note that this may lead to + undefined interpreter behaviour if exceptions raised by the trace + function escape to the function being traced. + + * - .. attribute:: frame.f_lineno + - The current line number of the frame -- writing to this + from within a trace function jumps to the given line (only for the bottom-most + frame). A debugger can implement a Jump command (aka Set Next Statement) + by writing to this attribute. + +Frame object methods +~~~~~~~~~~~~~~~~~~~~ + +Frame objects support one method: + +.. method:: frame.clear() + + This method clears all references to :ref:`local variables ` held by the + frame. Also, if the frame belonged to a :term:`generator`, the generator + is finalized. This helps break reference cycles involving frame + objects (for example when catching an :ref:`exception ` + and storing its :ref:`traceback ` for later use). + + :exc:`RuntimeError` is raised if the frame is currently executing + or suspended. + + .. versionadded:: 3.4 + + .. versionchanged:: 3.13 + Attempting to clear a suspended frame raises :exc:`RuntimeError` + (as has always been the case for executing frames). + + +.. _traceback-objects: + +Traceback objects +^^^^^^^^^^^^^^^^^ + +.. index:: + pair: object; traceback + pair: stack; trace + pair: exception; handler + pair: execution; stack + single: exc_info (in module sys) + single: last_traceback (in module sys) + single: sys.exc_info + single: sys.exception + single: sys.last_traceback + +Traceback objects represent a stack trace of an exception. A traceback object +is implicitly created when an exception occurs, and may also be explicitly +created by calling :class:`types.TracebackType`. + +For implicitly created tracebacks, when the search for an exception handler +unwinds the execution stack, at each unwound level a traceback object is +inserted in front of the current traceback. When an exception handler is +entered, the stack trace is made available to the program. (See section +:ref:`try`.) It is accessible as the third item of the +tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute +of the caught exception. + +When the program contains no suitable +handler, the stack trace is written (nicely formatted) to the standard error +stream; if the interpreter is interactive, it is also made available to the user +as ``sys.last_traceback``. + +For explicitly created tracebacks, it is up to the creator of the traceback +to determine how the ``tb_next`` attributes should be linked to form a +full stack trace. + +.. index:: + single: tb_frame (traceback attribute) + single: tb_lineno (traceback attribute) + single: tb_lasti (traceback attribute) + pair: statement; try + +Special read-only attributes: +:attr:`tb_frame` points to the execution frame of the current level; +:attr:`tb_lineno` gives the line number where the exception occurred; +:attr:`tb_lasti` indicates the precise instruction. +The line number and last instruction in the traceback may differ from the +line number of its frame object if the exception occurred in a +:keyword:`try` statement with no matching except clause or with a +finally clause. + +Accessing ``tb_frame`` raises an :ref:`auditing event ` +``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. + +.. index:: + single: tb_next (traceback attribute) + +Special writable attribute: :attr:`tb_next` is the next level in the stack +trace (towards the frame where the exception occurred), or ``None`` if +there is no next level. + +.. versionchanged:: 3.7 + Traceback objects can now be explicitly instantiated from Python code, + and the ``tb_next`` attribute of existing instances can be updated. + + +Slice objects +^^^^^^^^^^^^^ + +.. index:: pair: built-in function; slice + +Slice objects are used to represent slices for +:meth:`~object.__getitem__` +methods. They are also created by the built-in :func:`slice` function. + +.. index:: + single: start (slice object attribute) + single: stop (slice object attribute) + single: step (slice object attribute) + +Special read-only attributes: :attr:`~slice.start` is the lower bound; +:attr:`~slice.stop` is the upper bound; :attr:`~slice.step` is the step +value; each is ``None`` if omitted. These attributes can have any type. + +Slice objects support one method: + +.. method:: slice.indices(self, length) + + This method takes a single integer argument *length* and computes + information about the slice that the slice object would describe if + applied to a sequence of *length* items. It returns a tuple of three + integers; respectively these are the *start* and *stop* indices and the + *step* or stride length of the slice. Missing or out-of-bounds indices + are handled in a manner consistent with regular slices. + + +Static method objects +^^^^^^^^^^^^^^^^^^^^^ + +Static method objects provide a way of defeating the transformation of function +objects to method objects described above. A static method object is a wrapper +around any other object, usually a user-defined method object. When a static +method object is retrieved from a class or a class instance, the object actually +returned is the wrapped object, which is not subject to any further +transformation. Static method objects are also callable. Static method +objects are created by the built-in :func:`staticmethod` constructor. + + +Class method objects +^^^^^^^^^^^^^^^^^^^^ + +A class method object, like a static method object, is a wrapper around another +object that alters the way in which that object is retrieved from classes and +class instances. The behaviour of class method objects upon such retrieval is +described above, under "User-defined methods". Class method objects are created +by the built-in :func:`classmethod` constructor. .. _specialnames: @@ -1582,9 +1822,9 @@ Basic customization Called to implement truth value testing and the built-in operation ``bool()``; should return ``False`` or ``True``. When this method is not - defined, :meth:`__len__` is called, if it is defined, and the object is + defined, :meth:`~object.__len__` is called, if it is defined, and the object is considered true if its result is nonzero. If a class defines neither - :meth:`__len__` nor :meth:`__bool__`, all its instances are considered + :meth:`!__len__` nor :meth:`!__bool__`, all its instances are considered true. @@ -1613,7 +1853,7 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. :meth:`__getattr__` and :meth:`__setattr__`.) This is done both for efficiency reasons and because otherwise :meth:`__getattr__` would have no way to access other attributes of the instance. Note that at least for instance variables, - you can fake total control by not inserting any values in the instance attribute + you can take total control by not inserting any values in the instance attribute dictionary (but instead inserting them in another object). See the :meth:`__getattribute__` method below for a way to actually get total control over attribute access. @@ -1778,13 +2018,17 @@ class' :attr:`~object.__dict__`. Called to delete the attribute on an instance *instance* of the owner class. +Instances of descriptors may also have the :attr:`!__objclass__` attribute +present: + +.. attribute:: object.__objclass__ -The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module -as specifying the class where this object was defined (setting this -appropriately can assist in runtime introspection of dynamic class attributes). -For callables, it may indicate that an instance of the given type (or a -subclass) is expected or required as the first positional argument (for example, -CPython sets this attribute for unbound methods that are implemented in C). + The attribute :attr:`!__objclass__` is interpreted by the :mod:`inspect` module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class attributes). + For callables, it may indicate that an instance of the given type (or a + subclass) is expected or required as the first positional argument (for example, + CPython sets this attribute for unbound methods that are implemented in C). .. _descriptor-invocation: @@ -1865,13 +2109,14 @@ For instance bindings, the precedence of descriptor invocation depends on which descriptor methods are defined. A descriptor can define any combination of :meth:`~object.__get__`, :meth:`~object.__set__` and :meth:`~object.__delete__`. If it does not -define :meth:`__get__`, then accessing the attribute will return the descriptor +define :meth:`!__get__`, then accessing the attribute will return the descriptor object itself unless there is a value in the object's instance dictionary. If -the descriptor defines :meth:`__set__` and/or :meth:`__delete__`, it is a data +the descriptor defines :meth:`!__set__` and/or :meth:`!__delete__`, it is a data descriptor; if it defines neither, it is a non-data descriptor. Normally, data -descriptors define both :meth:`__get__` and :meth:`__set__`, while non-data -descriptors have just the :meth:`__get__` method. Data descriptors with -:meth:`__get__` and :meth:`__set__` (and/or :meth:`__delete__`) defined always override a redefinition in an +descriptors define both :meth:`!__get__` and :meth:`!__set__`, while non-data +descriptors have just the :meth:`!__get__` method. Data descriptors with +:meth:`!__get__` and :meth:`!__set__` (and/or :meth:`!__delete__`) defined +always override a redefinition in an instance dictionary. In contrast, non-data descriptors can be overridden by instances. @@ -1908,8 +2153,7 @@ Attribute lookup speed can be significantly improved as well. .. _datamodel-note-slots: -Notes on using *__slots__* -"""""""""""""""""""""""""" +Notes on using *__slots__*: * When inheriting from a class without *__slots__*, the :attr:`~object.__dict__` and @@ -2449,16 +2693,17 @@ either to emulate a sequence or to emulate a mapping; the difference is that for a sequence, the allowable keys should be the integers *k* for which ``0 <= k < N`` where *N* is the length of the sequence, or :class:`slice` objects, which define a range of items. It is also recommended that mappings provide the methods -:meth:`keys`, :meth:`values`, :meth:`items`, :meth:`get`, :meth:`clear`, -:meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`!copy`, and -:meth:`update` behaving similar to those for Python's standard :class:`dictionary ` +:meth:`!keys`, :meth:`!values`, :meth:`!items`, :meth:`!get`, :meth:`!clear`, +:meth:`!setdefault`, :meth:`!pop`, :meth:`!popitem`, :meth:`!copy`, and +:meth:`!update` behaving similar to those for Python's standard :class:`dictionary ` objects. The :mod:`collections.abc` module provides a :class:`~collections.abc.MutableMapping` :term:`abstract base class` to help create those methods from a base set of -:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__delitem__`, and :meth:`keys`. -Mutable sequences should provide methods :meth:`append`, :meth:`count`, -:meth:`index`, :meth:`extend`, :meth:`insert`, :meth:`pop`, :meth:`remove`, -:meth:`reverse` and :meth:`sort`, like Python standard :class:`list` +:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, +:meth:`~object.__delitem__`, and :meth:`!keys`. +Mutable sequences should provide methods :meth:`!append`, :meth:`!count`, +:meth:`!index`, :meth:`!extend`, :meth:`!insert`, :meth:`!pop`, :meth:`!remove`, +:meth:`!reverse` and :meth:`!sort`, like Python standard :class:`list` objects. Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods @@ -2471,7 +2716,7 @@ operator; for mappings, ``in`` should search the mapping's keys; for sequences, it should search through the values. It is further recommended that both mappings and sequences implement the :meth:`~object.__iter__` method to allow efficient iteration -through the container; for mappings, :meth:`__iter__` should iterate +through the container; for mappings, :meth:`!__iter__` should iterate through the object's keys; for sequences, it should iterate through the values. .. method:: object.__len__(self) @@ -2482,16 +2727,16 @@ through the object's keys; for sequences, it should iterate through the values. Called to implement the built-in function :func:`len`. Should return the length of the object, an integer ``>=`` 0. Also, an object that doesn't define a - :meth:`__bool__` method and whose :meth:`__len__` method returns zero is + :meth:`~object.__bool__` method and whose :meth:`!__len__` method returns zero is considered to be false in a Boolean context. .. impl-detail:: - In CPython, the length is required to be at most :attr:`sys.maxsize`. - If the length is larger than :attr:`!sys.maxsize` some features (such as + In CPython, the length is required to be at most :data:`sys.maxsize`. + If the length is larger than :data:`!sys.maxsize` some features (such as :func:`len`) may raise :exc:`OverflowError`. To prevent raising :exc:`!OverflowError` by truth value testing, an object must define a - :meth:`__bool__` method. + :meth:`~object.__bool__` method. .. method:: object.__length_hint__(self) @@ -2822,7 +3067,7 @@ For more information on context managers, see :ref:`typecontextmanager`. (i.e., prevent it from being propagated), it should return a true value. Otherwise, the exception will be processed normally upon exit from this method. - Note that :meth:`__exit__` methods should not reraise the passed-in exception; + Note that :meth:`~object.__exit__` methods should not reraise the passed-in exception; this is the caller's responsibility. @@ -3050,7 +3295,7 @@ generators, coroutines do not directly support iteration. to the :meth:`~generator.send` method of the iterator that caused the coroutine to suspend. The result (return value, :exc:`StopIteration`, or other exception) is the same as when - iterating over the :meth:`__await__` return value, described above. + iterating over the :meth:`!__await__` return value, described above. .. method:: coroutine.throw(value) coroutine.throw(type[, value[, traceback]]) @@ -3140,12 +3385,12 @@ Asynchronous context managers can be used in an :keyword:`async with` statement. .. method:: object.__aenter__(self) - Semantically similar to :meth:`__enter__`, the only + Semantically similar to :meth:`~object.__enter__`, the only difference being that it must return an *awaitable*. .. method:: object.__aexit__(self, exc_type, exc_value, traceback) - Semantically similar to :meth:`__exit__`, the only + Semantically similar to :meth:`~object.__exit__`, the only difference being that it must return an *awaitable*. An example of an asynchronous context manager class:: @@ -3167,8 +3412,9 @@ An example of an asynchronous context manager class:: lead to some very strange behaviour if it is handled incorrectly. .. [#] The :meth:`~object.__hash__`, :meth:`~object.__iter__`, - :meth:`~object.__reversed__`, and :meth:`~object.__contains__` methods have - special handling for this; others + :meth:`~object.__reversed__`, :meth:`~object.__contains__`, + :meth:`~object.__class_getitem__` and :meth:`~os.PathLike.__fspath__` + methods have special handling for this. Others will still raise a :exc:`TypeError`, but may do so by relying on the behavior that ``None`` is not callable. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 8917243999d399..cea3a56ba51644 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -71,6 +71,8 @@ The following constructs bind names: + in a capture pattern in structural pattern matching * :keyword:`import` statements. +* :keyword:`type` statements. +* :ref:`type parameter lists `. The :keyword:`!import` statement of the form ``from ... import *`` binds all names defined in the imported module, except those beginning with an underscore. @@ -149,7 +151,8 @@ a global statement, the free variable is treated as a global. The :keyword:`nonlocal` statement causes corresponding names to refer to previously bound variables in the nearest enclosing function scope. :exc:`SyntaxError` is raised at compile time if the given name does not -exist in any enclosing function scope. +exist in any enclosing function scope. :ref:`Type parameters ` +cannot be rebound with the :keyword:`!nonlocal` statement. .. index:: pair: module; __main__ @@ -163,14 +166,119 @@ These references follow the normal rules for name resolution with an exception that unbound local variables are looked up in the global namespace. The namespace of the class definition becomes the attribute dictionary of the class. The scope of names defined in a class block is limited to the -class block; it does not extend to the code blocks of methods -- this includes -comprehensions and generator expressions since they are implemented using a -function scope. This means that the following will fail:: +class block; it does not extend to the code blocks of methods. This includes +comprehensions and generator expressions, but it does not include +:ref:`annotation scopes `, +which have access to their enclosing class scopes. +This means that the following will fail:: class A: a = 42 b = list(a + i for i in range(10)) +However, the following will succeed:: + + class A: + type Alias = Nested + class Nested: pass + + print(A.Alias.__value__) # + +.. _annotation-scopes: + +Annotation scopes +----------------- + +:ref:`Type parameter lists ` and :keyword:`type` statements +introduce *annotation scopes*, which behave mostly like function scopes, +but with some exceptions discussed below. :term:`Annotations ` +currently do not use annotation scopes, but they are expected to use +annotation scopes in Python 3.13 when :pep:`649` is implemented. + +Annotation scopes are used in the following contexts: + +* Type parameter lists for :ref:`generic type aliases `. +* Type parameter lists for :ref:`generic functions `. + A generic function's annotations are + executed within the annotation scope, but its defaults and decorators are not. +* Type parameter lists for :ref:`generic classes `. + A generic class's base classes and + keyword arguments are executed within the annotation scope, but its decorators are not. +* The bounds and constraints for type variables + (:ref:`lazily evaluated `). +* The value of type aliases (:ref:`lazily evaluated `). + +Annotation scopes differ from function scopes in the following ways: + +* Annotation scopes have access to their enclosing class namespace. + If an annotation scope is immediately within a class scope, or within another + annotation scope that is immediately within a class scope, the code in the + annotation scope can use names defined in the class scope as if it were + executed directly within the class body. This contrasts with regular + functions defined within classes, which cannot access names defined in the class scope. +* Expressions in annotation scopes cannot contain :keyword:`yield`, ``yield from``, + :keyword:`await`, or :token:`:= ` + expressions. (These expressions are allowed in other scopes contained within the + annotation scope.) +* Names defined in annotation scopes cannot be rebound with :keyword:`nonlocal` + statements in inner scopes. This includes only type parameters, as no other + syntactic elements that can appear within annotation scopes can introduce new names. +* While annotation scopes have an internal name, that name is not reflected in the + :term:`__qualname__ ` of objects defined within the scope. + Instead, the :attr:`!__qualname__` + of such objects is as if the object were defined in the enclosing scope. + +.. versionadded:: 3.12 + Annotation scopes were introduced in Python 3.12 as part of :pep:`695`. + +.. _lazy-evaluation: + +Lazy evaluation +--------------- + +The values of type aliases created through the :keyword:`type` statement are +*lazily evaluated*. The same applies to the bounds and constraints of type +variables created through the :ref:`type parameter syntax `. +This means that they are not evaluated when the type alias or type variable is +created. Instead, they are only evaluated when doing so is necessary to resolve +an attribute access. + +Example: + +.. doctest:: + + >>> type Alias = 1/0 + >>> Alias.__value__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def func[T: 1/0](): pass + >>> T = func.__type_params__[0] + >>> T.__bound__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + +Here the exception is raised only when the ``__value__`` attribute +of the type alias or the ``__bound__`` attribute of the type variable +is accessed. + +This behavior is primarily useful for references to types that have not +yet been defined when the type alias or type variable is created. For example, +lazy evaluation enables creation of mutually recursive type aliases:: + + from typing import Literal + + type SimpleExpr = int | Parenthesized + type Parenthesized = tuple[Literal["("], Expr, Literal[")"]] + type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", "-"], Expr] + +Lazily evaluated values are evaluated in :ref:`annotation scope `, +which means that names that appear inside the lazily evaluated value are looked up +as if they were used in the immediately enclosing scope. + +.. versionadded:: 3.12 + .. _restrict_exec: Builtins and restricted execution diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b97a08f25d92a2..3f6d5bfafee9d1 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -298,27 +298,27 @@ Dictionary displays .. index:: pair: dictionary; display pair: dictionary; comprehensions - key, datum, key/datum pair + key, value, key/value pair pair: object; dictionary single: {} (curly brackets); dictionary expression single: : (colon); in dictionary expressions single: , (comma); in dictionary displays -A dictionary display is a possibly empty series of key/datum pairs enclosed in -curly braces: +A dictionary display is a possibly empty series of dict items (key/value pairs) +enclosed in curly braces: .. productionlist:: python-grammar - dict_display: "{" [`key_datum_list` | `dict_comprehension`] "}" - key_datum_list: `key_datum` ("," `key_datum`)* [","] - key_datum: `expression` ":" `expression` | "**" `or_expr` + dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" + dict_item_list: `dict_item` ("," `dict_item`)* [","] + dict_item: `expression` ":" `expression` | "**" `or_expr` dict_comprehension: `expression` ":" `expression` `comp_for` A dictionary display yields a new dictionary object. -If a comma-separated sequence of key/datum pairs is given, they are evaluated +If a comma-separated sequence of dict items is given, they are evaluated from left to right to define the entries of the dictionary: each key object is -used as a key into the dictionary to store the corresponding datum. This means -that you can specify the same key multiple times in the key/datum list, and the +used as a key into the dictionary to store the corresponding value. This means +that you can specify the same key multiple times in the dict item list, and the final dictionary's value for that key will be the last one given. .. index:: @@ -328,7 +328,7 @@ final dictionary's value for that key will be the last one given. A double asterisk ``**`` denotes :dfn:`dictionary unpacking`. Its operand must be a :term:`mapping`. Each mapping item is added to the new dictionary. Later values replace values already set by -earlier key/datum pairs and earlier dictionary unpackings. +earlier dict items and earlier dictionary unpackings. .. versionadded:: 3.5 Unpacking into dictionary displays, originally proposed by :pep:`448`. @@ -344,7 +344,7 @@ in the new dictionary in the order they are produced. Restrictions on the types of the key values are listed earlier in section :ref:`types`. (To summarize, the key type should be :term:`hashable`, which excludes all mutable objects.) Clashes between duplicate keys are not detected; the last -datum (textually rightmost in the display) stored for a given key value +value (textually rightmost in the display) stored for a given key value prevails. .. versionchanged:: 3.8 @@ -499,8 +499,8 @@ the yield expression. It can be either set explicitly when raising :exc:`StopIteration`, or automatically when the subiterator is a generator (by returning a value from the subgenerator). - .. versionchanged:: 3.3 - Added ``yield from `` to delegate control flow to a subiterator. +.. versionchanged:: 3.3 + Added ``yield from `` to delegate control flow to a subiterator. The parentheses may be omitted when the yield expression is the sole expression on the right hand side of an assignment statement. @@ -595,12 +595,19 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.close() Raises a :exc:`GeneratorExit` at the point where the generator function was - paused. If the generator function then exits gracefully, is already closed, - or raises :exc:`GeneratorExit` (by not catching the exception), close - returns to its caller. If the generator yields a value, a - :exc:`RuntimeError` is raised. If the generator raises any other exception, - it is propagated to the caller. :meth:`close` does nothing if the generator - has already exited due to an exception or normal exit. + paused. If the generator function catches the exception and returns a + value, this value is returned from :meth:`close`. If the generator function + is already closed, or raises :exc:`GeneratorExit` (by not catching the + exception), :meth:`close` returns :const:`None`. If the generator yields a + value, a :exc:`RuntimeError` is raised. If the generator raises any other + exception, it is propagated to the caller. If the generator has already + exited due to an exception or normal exit, :meth:`close` returns + :const:`None` and has no other effect. + + .. versionchanged:: 3.13 + + If a generator returns a value upon being closed, the value is returned + by :meth:`close`. .. index:: single: yield; examples @@ -816,12 +823,18 @@ An attribute reference is a primary followed by a period and a name: The primary must evaluate to an object of a type that supports attribute references, which most objects do. This object is then asked to produce the -attribute whose name is the identifier. This production can be customized by -overriding the :meth:`__getattr__` method. If this attribute is not available, -the exception :exc:`AttributeError` is raised. Otherwise, the type and value of -the object produced is determined by the object. Multiple evaluations of the -same attribute reference may yield different objects. +attribute whose name is the identifier. The type and value produced is +determined by the object. Multiple evaluations of the same attribute +reference may yield different objects. + +This production can be customized by overriding the +:meth:`~object.__getattribute__` method or the :meth:`~object.__getattr__` +method. The :meth:`!__getattribute__` method is called first and either +returns a value or raises :exc:`AttributeError` if the attribute is not +available. +If an :exc:`AttributeError` is raised and the object has a :meth:`!__getattr__` +method, that method is called as a fallback. .. _subscriptions: @@ -882,7 +895,7 @@ to the index so that, for example, ``x[-1]`` selects the last item of ``x``. The resulting value must be a nonnegative integer less than the number of items in the sequence, and the subscription selects the item whose index is that value (counting from zero). Since the support for negative indices and slicing -occurs in the object's :meth:`__getitem__` method, subclasses overriding +occurs in the object's :meth:`~object.__getitem__` method, subclasses overriding this method will need to explicitly add that support. .. index:: @@ -937,7 +950,7 @@ slice list contains no proper slice). single: step (slice object attribute) The semantics for a slicing are as follows. The primary is indexed (using the -same :meth:`__getitem__` method as +same :meth:`~object.__getitem__` method as normal subscription) with a key that is constructed from the slice list, as follows. If the slice list contains at least one comma, the key is a tuple containing the conversion of the slice items; otherwise, the conversion of the @@ -1663,7 +1676,7 @@ If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative +:meth:`~object.__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x is y[i] or x == y[i]``, and no lower integer index raises the :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). @@ -1717,7 +1730,7 @@ control flow statements, the following values are interpreted as false: ``False``, ``None``, numeric zero of all types, and empty strings and containers (including strings, tuples, lists, dictionaries, sets and frozensets). All other values are interpreted as true. User-defined objects can customize their -truth value by providing a :meth:`__bool__` method. +truth value by providing a :meth:`~object.__bool__` method. .. index:: pair: operator; not @@ -1774,10 +1787,11 @@ Or, when processing a file stream in chunks: while chunk := file.read(9000): process(chunk) -Assignment expressions must be surrounded by parentheses when used -as sub-expressions in slicing, conditional, lambda, -keyword-argument, and comprehension-if expressions -and in ``assert`` and ``with`` statements. +Assignment expressions must be surrounded by parentheses when +used as expression statements and when used as sub-expressions in +slicing, conditional, lambda, +keyword-argument, and comprehension-if expressions and +in ``assert``, ``with``, and ``assignment`` statements. In all other places where they can be used, parentheses are not required, including in ``if`` and ``while`` statements. diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 0f416a5c583f85..a7beeea29b4556 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -375,32 +375,32 @@ of what happens during the loading portion of import:: Note the following details: - * If there is an existing module object with the given name in - :data:`sys.modules`, import will have already returned it. +* If there is an existing module object with the given name in + :data:`sys.modules`, import will have already returned it. - * The module will exist in :data:`sys.modules` before the loader - executes the module code. This is crucial because the module code may - (directly or indirectly) import itself; adding it to :data:`sys.modules` - beforehand prevents unbounded recursion in the worst case and multiple - loading in the best. +* The module will exist in :data:`sys.modules` before the loader + executes the module code. This is crucial because the module code may + (directly or indirectly) import itself; adding it to :data:`sys.modules` + beforehand prevents unbounded recursion in the worst case and multiple + loading in the best. - * If loading fails, the failing module -- and only the failing module -- - gets removed from :data:`sys.modules`. Any module already in the - :data:`sys.modules` cache, and any module that was successfully loaded - as a side-effect, must remain in the cache. This contrasts with - reloading where even the failing module is left in :data:`sys.modules`. +* If loading fails, the failing module -- and only the failing module -- + gets removed from :data:`sys.modules`. Any module already in the + :data:`sys.modules` cache, and any module that was successfully loaded + as a side-effect, must remain in the cache. This contrasts with + reloading where even the failing module is left in :data:`sys.modules`. - * After the module is created but before execution, the import machinery - sets the import-related module attributes ("_init_module_attrs" in - the pseudo-code example above), as summarized in a - :ref:`later section `. +* After the module is created but before execution, the import machinery + sets the import-related module attributes ("_init_module_attrs" in + the pseudo-code example above), as summarized in a + :ref:`later section `. - * Module execution is the key moment of loading in which the module's - namespace gets populated. Execution is entirely delegated to the - loader, which gets to decide what gets populated and how. +* Module execution is the key moment of loading in which the module's + namespace gets populated. Execution is entirely delegated to the + loader, which gets to decide what gets populated and how. - * The module created during loading and passed to exec_module() may - not be the one returned at the end of import [#fnlo]_. +* The module created during loading and passed to exec_module() may + not be the one returned at the end of import [#fnlo]_. .. versionchanged:: 3.4 The import system has taken over the boilerplate responsibilities of @@ -417,13 +417,13 @@ returned from :meth:`~importlib.abc.Loader.exec_module` is ignored. Loaders must satisfy the following requirements: - * If the module is a Python module (as opposed to a built-in module or a - dynamically loaded extension), the loader should execute the module's code - in the module's global name space (``module.__dict__``). +* If the module is a Python module (as opposed to a built-in module or a + dynamically loaded extension), the loader should execute the module's code + in the module's global name space (``module.__dict__``). - * If the loader cannot execute the module, it should raise an - :exc:`ImportError`, although any other exception raised during - :meth:`~importlib.abc.Loader.exec_module` will be propagated. +* If the loader cannot execute the module, it should raise an + :exc:`ImportError`, although any other exception raised during + :meth:`~importlib.abc.Loader.exec_module` will be propagated. In many cases, the finder and loader can be the same object; in such cases the :meth:`~importlib.abc.MetaPathFinder.find_spec` method would just return a @@ -453,20 +453,20 @@ import machinery will create the new module itself. functionality described above in addition to executing the module. All the same constraints apply, with some additional clarification: - * If there is an existing module object with the given name in - :data:`sys.modules`, the loader must use that existing module. - (Otherwise, :func:`importlib.reload` will not work correctly.) If the - named module does not exist in :data:`sys.modules`, the loader - must create a new module object and add it to :data:`sys.modules`. + * If there is an existing module object with the given name in + :data:`sys.modules`, the loader must use that existing module. + (Otherwise, :func:`importlib.reload` will not work correctly.) If the + named module does not exist in :data:`sys.modules`, the loader + must create a new module object and add it to :data:`sys.modules`. - * The module *must* exist in :data:`sys.modules` before the loader - executes the module code, to prevent unbounded recursion or multiple - loading. + * The module *must* exist in :data:`sys.modules` before the loader + executes the module code, to prevent unbounded recursion or multiple + loading. - * If loading fails, the loader must remove any modules it has inserted - into :data:`sys.modules`, but it must remove **only** the failing - module(s), and only if the loader itself has loaded the module(s) - explicitly. + * If loading fails, the loader must remove any modules it has inserted + into :data:`sys.modules`, but it must remove **only** the failing + module(s), and only if the loader itself has loaded the module(s) + explicitly. .. versionchanged:: 3.5 A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but @@ -559,7 +559,7 @@ listed below. functionality, for example getting data associated with a loader. It is **strongly** recommended that you rely on :attr:`__spec__` - instead instead of this attribute. + instead of this attribute. .. versionchanged:: 3.12 The value of ``__loader__`` is expected to be the same as @@ -580,7 +580,7 @@ listed below. relative imports for main modules, as defined in :pep:`366`. It is **strongly** recommended that you rely on :attr:`__spec__` - instead instead of this attribute. + instead of this attribute. .. versionchanged:: 3.6 The value of ``__package__`` is expected to be the same as @@ -650,7 +650,7 @@ listed below. from a file, that atypical scenario may be appropriate. It is **strongly** recommended that you rely on :attr:`__spec__` - instead instead of ``__cached__``. + instead of ``__cached__``. .. _package-path-rules: @@ -693,17 +693,17 @@ with defaults for whatever information is missing. Here are the exact rules used: - * If the module has a ``__spec__`` attribute, the information in the spec - is used to generate the repr. The "name", "loader", "origin", and - "has_location" attributes are consulted. +* If the module has a ``__spec__`` attribute, the information in the spec + is used to generate the repr. The "name", "loader", "origin", and + "has_location" attributes are consulted. - * If the module has a ``__file__`` attribute, this is used as part of the - module's repr. +* If the module has a ``__file__`` attribute, this is used as part of the + module's repr. - * If the module has no ``__file__`` but does have a ``__loader__`` that is not - ``None``, then the loader's repr is used as part of the module's repr. +* If the module has no ``__file__`` but does have a ``__loader__`` that is not + ``None``, then the loader's repr is used as part of the module's repr. - * Otherwise, just use the module's ``__name__`` in the repr. +* Otherwise, just use the module's ``__name__`` in the repr. .. versionchanged:: 3.12 Use of :meth:`!module_repr`, having been deprecated since Python 3.4, was diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index 81f0a5c5d43883..cf186705e6e987 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -90,7 +90,8 @@ Notation .. index:: BNF, grammar, syntax, notation -The descriptions of lexical analysis and syntax use a modified BNF grammar +The descriptions of lexical analysis and syntax use a modified +`Backus–Naur form (BNF) `_ grammar notation. This uses the following style of definition: .. productionlist:: notation diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 8adb4b740825d0..3e07d16068a627 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -315,7 +315,7 @@ The Unicode category codes mentioned above stand for: * *Nd* - decimal numbers * *Pc* - connector punctuations * *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards + `_ to support backwards compatibility * *Other_ID_Continue* - likewise @@ -323,8 +323,8 @@ All identifiers are converted into the normal form NFKC while parsing; compariso of identifiers is based on NFKC. A non-normative HTML file listing all valid identifier characters for Unicode -15.0.0 can be found at -https://www.unicode.org/Public/15.0.0/ucd/DerivedCoreProperties.txt +15.1.0 can be found at +https://www.unicode.org/Public/15.1.0/ucd/DerivedCoreProperties.txt .. _keywords: @@ -361,15 +361,19 @@ Soft Keywords .. versionadded:: 3.10 Some identifiers are only reserved under specific contexts. These are known as -*soft keywords*. The identifiers ``match``, ``case`` and ``_`` can -syntactically act as keywords in contexts related to the pattern matching -statement, but this distinction is done at the parser level, not when -tokenizing. +*soft keywords*. The identifiers ``match``, ``case``, ``type`` and ``_`` can +syntactically act as keywords in certain contexts, +but this distinction is done at the parser level, not when tokenizing. -As soft keywords, their use with pattern matching is possible while still -preserving compatibility with existing code that uses ``match``, ``case`` and ``_`` as +As soft keywords, their use in the grammar is possible while still +preserving compatibility with existing code that uses these names as identifier names. +``match``, ``case``, and ``_`` are used in the :keyword:`match` statement. +``type`` is used in the :keyword:`type` statement. + +.. versionchanged:: 3.12 + ``type`` is now a soft keyword. .. index:: single: _, identifiers @@ -545,55 +549,59 @@ retained), except that three unescaped quotes in a row terminate the literal. ( .. _escape-sequences: + +Escape sequences +^^^^^^^^^^^^^^^^ + Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and bytes literals are interpreted according to rules similar to those used by Standard C. The recognized escape sequences are: -+-----------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=================+=================================+=======+ -| ``\``\ | Backslash and newline ignored | \(1) | -+-----------------+---------------------------------+-------+ -| ``\\`` | Backslash (``\``) | | -+-----------------+---------------------------------+-------+ -| ``\'`` | Single quote (``'``) | | -+-----------------+---------------------------------+-------+ -| ``\"`` | Double quote (``"``) | | -+-----------------+---------------------------------+-------+ -| ``\a`` | ASCII Bell (BEL) | | -+-----------------+---------------------------------+-------+ -| ``\b`` | ASCII Backspace (BS) | | -+-----------------+---------------------------------+-------+ -| ``\f`` | ASCII Formfeed (FF) | | -+-----------------+---------------------------------+-------+ -| ``\n`` | ASCII Linefeed (LF) | | -+-----------------+---------------------------------+-------+ -| ``\r`` | ASCII Carriage Return (CR) | | -+-----------------+---------------------------------+-------+ -| ``\t`` | ASCII Horizontal Tab (TAB) | | -+-----------------+---------------------------------+-------+ -| ``\v`` | ASCII Vertical Tab (VT) | | -+-----------------+---------------------------------+-------+ -| ``\ooo`` | Character with octal value | (2,4) | -| | *ooo* | | -+-----------------+---------------------------------+-------+ -| ``\xhh`` | Character with hex value *hh* | (3,4) | -+-----------------+---------------------------------+-------+ ++-------------------------+---------------------------------+-------+ +| Escape Sequence | Meaning | Notes | ++=========================+=================================+=======+ +| ``\``\ | Backslash and newline ignored | \(1) | ++-------------------------+---------------------------------+-------+ +| ``\\`` | Backslash (``\``) | | ++-------------------------+---------------------------------+-------+ +| ``\'`` | Single quote (``'``) | | ++-------------------------+---------------------------------+-------+ +| ``\"`` | Double quote (``"``) | | ++-------------------------+---------------------------------+-------+ +| ``\a`` | ASCII Bell (BEL) | | ++-------------------------+---------------------------------+-------+ +| ``\b`` | ASCII Backspace (BS) | | ++-------------------------+---------------------------------+-------+ +| ``\f`` | ASCII Formfeed (FF) | | ++-------------------------+---------------------------------+-------+ +| ``\n`` | ASCII Linefeed (LF) | | ++-------------------------+---------------------------------+-------+ +| ``\r`` | ASCII Carriage Return (CR) | | ++-------------------------+---------------------------------+-------+ +| ``\t`` | ASCII Horizontal Tab (TAB) | | ++-------------------------+---------------------------------+-------+ +| ``\v`` | ASCII Vertical Tab (VT) | | ++-------------------------+---------------------------------+-------+ +| :samp:`\\\\{ooo}` | Character with octal value | (2,4) | +| | *ooo* | | ++-------------------------+---------------------------------+-------+ +| :samp:`\\x{hh}` | Character with hex value *hh* | (3,4) | ++-------------------------+---------------------------------+-------+ Escape sequences only recognized in string literals are: -+-----------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=================+=================================+=======+ -| ``\N{name}`` | Character named *name* in the | \(5) | -| | Unicode database | | -+-----------------+---------------------------------+-------+ -| ``\uxxxx`` | Character with 16-bit hex value | \(6) | -| | *xxxx* | | -+-----------------+---------------------------------+-------+ -| ``\Uxxxxxxxx`` | Character with 32-bit hex value | \(7) | -| | *xxxxxxxx* | | -+-----------------+---------------------------------+-------+ ++-------------------------+---------------------------------+-------+ +| Escape Sequence | Meaning | Notes | ++=========================+=================================+=======+ +| :samp:`\\N\\{{name}\\}` | Character named *name* in the | \(5) | +| | Unicode database | | ++-------------------------+---------------------------------+-------+ +| :samp:`\\u{xxxx}` | Character with 16-bit hex value | \(6) | +| | *xxxx* | | ++-------------------------+---------------------------------+-------+ +| :samp:`\\U{xxxxxxxx}` | Character with 32-bit hex value | \(7) | +| | *xxxxxxxx* | | ++-------------------------+---------------------------------+-------+ Notes: @@ -649,12 +657,12 @@ is more easily recognized as broken.) It is also important to note that the escape sequences only recognized in string literals fall into the category of unrecognized escapes for bytes literals. - .. versionchanged:: 3.6 - Unrecognized escape sequences produce a :exc:`DeprecationWarning`. +.. versionchanged:: 3.6 + Unrecognized escape sequences produce a :exc:`DeprecationWarning`. - .. versionchanged:: 3.12 - Unrecognized escape sequences produce a :exc:`SyntaxWarning`. In a future - Python version they will be eventually a :exc:`SyntaxError`. +.. versionchanged:: 3.12 + Unrecognized escape sequences produce a :exc:`SyntaxWarning`. In a future + Python version they will be eventually a :exc:`SyntaxError`. Even in a raw literal, quotes can be escaped with a backslash, but the backslash remains in the result; for example, ``r"\""`` is a valid string @@ -741,16 +749,28 @@ Expressions in formatted string literals are treated like regular Python expressions surrounded by parentheses, with a few exceptions. An empty expression is not allowed, and both :keyword:`lambda` and assignment expressions ``:=`` must be surrounded by explicit parentheses. -Replacement expressions can contain line breaks (e.g. in triple-quoted -strings), but they cannot contain comments. Each expression is evaluated -in the context where the formatted string literal appears, in order from -left to right. +Each expression is evaluated in the context where the formatted string literal +appears, in order from left to right. Replacement expressions can contain +newlines in both single-quoted and triple-quoted f-strings and they can contain +comments. Everything that comes after a ``#`` inside a replacement field +is a comment (even closing braces and quotes). In that case, replacement fields +must be closed in a different line. + +.. code-block:: text + + >>> f"abc{a # This is a comment }" + ... + 3}" + 'abc5' .. versionchanged:: 3.7 Prior to Python 3.7, an :keyword:`await` expression and comprehensions containing an :keyword:`async for` clause were illegal in the expressions in formatted string literals due to a problem with the implementation. +.. versionchanged:: 3.12 + Prior to Python 3.12, comments were not allowed inside f-string replacement + fields. + When the equal sign ``'='`` is provided, the output will have the expression text, the ``'='`` and the evaluated value. Spaces after the opening brace ``'{'``, within the expression and after the ``'='`` are all retained in the @@ -767,7 +787,7 @@ is converted before formatting. Conversion ``'!s'`` calls :func:`str` on the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. The result is then formatted using the :func:`format` protocol. The -format specifier is passed to the :meth:`__format__` method of the +format specifier is passed to the :meth:`~object.__format__` method of the expression or conversion result. An empty string is passed when the format specifier is omitted. The formatted result is then included in the final value of the whole string. @@ -813,24 +833,30 @@ Some examples of formatted string literals:: 'line = "The mill\'s closed" ' -A consequence of sharing the same syntax as regular string literals is -that characters in the replacement fields must not conflict with the -quoting used in the outer formatted string literal:: +Reusing the outer f-string quoting type inside a replacement field is +permitted:: - f"abc {a["x"]} def" # error: outer string literal ended prematurely - f"abc {a['x']} def" # workaround: use different quoting + >>> a = dict(x=2) + >>> f"abc {a["x"]} def" + 'abc 2 def' -Backslashes are not allowed in format expressions and will raise -an error:: +.. versionchanged:: 3.12 + Prior to Python 3.12, reuse of the same quoting type of the outer f-string + inside a replacement field was not possible. - f"newline: {ord('\n')}" # raises SyntaxError +Backslashes are also allowed in replacement fields and are evaluated the same +way as in any other context:: -To include a value in which a backslash escape is required, create -a temporary variable. + >>> a = ["a", "b", "c"] + >>> print(f"List a contains:\n{"\n".join(a)}") + List a contains: + a + b + c - >>> newline = ord('\n') - >>> f"newline: {newline}" - 'newline: 10' +.. versionchanged:: 3.12 + Prior to Python 3.12, backslashes were not permitted inside an f-string + replacement field. Formatted string literals cannot be used as docstrings, even if they do not include expressions. @@ -1019,4 +1045,4 @@ occurrence outside string literals and comments is an unconditional error: .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/15.0.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/15.1.0/ucd/NameAliases.txt diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index f7a8b44d195417..a9e65be1eda340 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -28,6 +28,7 @@ simple statements is: : | `future_stmt` : | `global_stmt` : | `nonlocal_stmt` + : | `type_stmt` .. _exprstmts: @@ -209,7 +210,7 @@ Assignment of an object to a single target is recursively defined as follows. If the primary is a mapping object (such as a dictionary), the subscript must have a type compatible with the mapping's key type, and the mapping is then - asked to create a key/datum pair which maps the subscript to the assigned + asked to create a key/value pair which maps the subscript to the assigned object. This can either replace an existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). @@ -1012,3 +1013,48 @@ pre-existing bindings in the local scope. :pep:`3104` - Access to Names in Outer Scopes The specification for the :keyword:`nonlocal` statement. + +.. _type: + +The :keyword:`!type` statement +============================== + +.. index:: pair: statement; type + +.. productionlist:: python-grammar + type_stmt: 'type' `identifier` [`type_params`] "=" `expression` + +The :keyword:`!type` statement declares a type alias, which is an instance +of :class:`typing.TypeAliasType`. + +For example, the following statement creates a type alias:: + + type Point = tuple[float, float] + +This code is roughly equivalent to:: + + annotation-def VALUE_OF_Point(): + return tuple[float, float] + Point = typing.TypeAliasType("Point", VALUE_OF_Point()) + +``annotation-def`` indicates an :ref:`annotation scope `, which behaves +mostly like a function, but with several small differences. + +The value of the +type alias is evaluated in the annotation scope. It is not evaluated when the +type alias is created, but only when the value is accessed through the type alias's +:attr:`!__value__` attribute (see :ref:`lazy-evaluation`). +This allows the type alias to refer to names that are not yet defined. + +Type aliases may be made generic by adding a :ref:`type parameter list ` +after the name. See :ref:`generic-type-aliases` for more. + +:keyword:`!type` is a :ref:`soft keyword `. + +.. versionadded:: 3.12 + +.. seealso:: + + :pep:`695` - Type Parameter Syntax + Introduced the :keyword:`!type` statement and syntax for + generic classes and functions. diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index d0390a04ea6dd8..597341d99ffeaa 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -7,32 +7,29 @@ blurb python-docs-theme>=2022.1 # Generated from: -# pip install "Sphinx~=3.2.0" "docutils<0.17" "Jinja2<3" "MarkupSafe<2" +# pip install "Sphinx~=4.2.0" # pip freeze # -# Sphinx 3.2 comes from ``needs_sphinx = '3.2'`` in ``Doc/conf.py``. -# Docutils<0.17, Jinja2<3, and MarkupSafe<2 are additionally specified as -# Sphinx 3.2 is incompatible with newer releases of these packages. +# Sphinx 4.2 comes from ``needs_sphinx = '4.2'`` in ``Doc/conf.py``. -Sphinx==3.2.1 alabaster==0.7.13 -Babel==2.12.1 -certifi==2022.12.7 -charset-normalizer==3.1.0 -colorama==0.4.6 -docutils==0.16 +Babel==2.13.0 +certifi==2023.7.22 +charset-normalizer==3.3.0 +docutils==0.17.1 idna==3.4 imagesize==1.4.1 -Jinja2==2.11.3 -MarkupSafe==1.1.1 -packaging==23.1 -Pygments==2.15.1 -requests==2.29.0 +Jinja2==3.1.2 +MarkupSafe==2.1.3 +packaging==23.2 +Pygments==2.16.1 +requests==2.31.0 snowballstemmer==2.2.0 +Sphinx==4.2.0 sphinxcontrib-applehelp==1.0.4 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 -urllib3==1.26.15 +urllib3==2.0.7 diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 9cbd15c2209dc6..04334fd5a464d4 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -6,15 +6,17 @@ # Sphinx version is pinned so that new versions that introduce new warnings # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. -sphinx==4.5.0 +# PR #104777: Sphinx 6.2 no longer uses imghdr, removed in Python 3.13. +sphinx==6.2.1 blurb -sphinx-lint==0.6.7 +sphinx-autobuild sphinxext-opengraph==0.7.5 +sphinx-notfound-page==1.0.0 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2022.1 +python-docs-theme>=2023.3.1,!=2023.7 -c constraints.txt diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1d3503bf06f085..5ef68cc089d436 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -1,130 +1,51 @@ # All RST files under Doc/ -- except these -- must pass Sphinx nit-picky mode, -# as tested on the CI via touch-clean-files.py in doc.yml. -# Add blank lines between files and keep them sorted lexicographically -# to help avoid merge conflicts. +# as tested on the CI via check-warnings.py in reusable-docs.yml. +# Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/allocation.rst -Doc/c-api/apiabiversion.rst -Doc/c-api/arg.rst -Doc/c-api/bool.rst -Doc/c-api/buffer.rst -Doc/c-api/bytes.rst -Doc/c-api/call.rst -Doc/c-api/capsule.rst -Doc/c-api/cell.rst -Doc/c-api/code.rst -Doc/c-api/codec.rst -Doc/c-api/complex.rst -Doc/c-api/conversion.rst -Doc/c-api/datetime.rst Doc/c-api/descriptor.rst -Doc/c-api/dict.rst Doc/c-api/exceptions.rst Doc/c-api/file.rst Doc/c-api/float.rst Doc/c-api/gcsupport.rst -Doc/c-api/import.rst Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst -Doc/c-api/iterator.rst -Doc/c-api/long.rst -Doc/c-api/mapping.rst -Doc/c-api/marshal.rst Doc/c-api/memory.rst Doc/c-api/memoryview.rst Doc/c-api/module.rst -Doc/c-api/none.rst Doc/c-api/object.rst -Doc/c-api/refcounting.rst -Doc/c-api/sequence.rst -Doc/c-api/set.rst Doc/c-api/stable.rst Doc/c-api/structures.rst Doc/c-api/sys.rst -Doc/c-api/tuple.rst Doc/c-api/type.rst -Doc/c-api/typehints.rst Doc/c-api/typeobj.rst -Doc/c-api/unicode.rst -Doc/c-api/veryhigh.rst -Doc/c-api/weakref.rst -Doc/extending/embedding.rst Doc/extending/extending.rst -Doc/extending/newtypes.rst -Doc/extending/newtypes_tutorial.rst -Doc/faq/design.rst -Doc/faq/extending.rst -Doc/faq/gui.rst -Doc/faq/library.rst -Doc/faq/programming.rst Doc/glossary.rst -Doc/howto/curses.rst Doc/howto/descriptor.rst Doc/howto/enum.rst -Doc/howto/functional.rst -Doc/howto/instrumentation.rst Doc/howto/isolating-extensions.rst -Doc/howto/logging-cookbook.rst Doc/howto/logging.rst -Doc/howto/regex.rst -Doc/howto/sorting.rst -Doc/howto/unicode.rst Doc/howto/urllib2.rst -Doc/install/index.rst -Doc/library/2to3.rst -Doc/library/__future__.rst -Doc/library/_thread.rst -Doc/library/abc.rst -Doc/library/aifc.rst Doc/library/ast.rst -Doc/library/asyncio-dev.rst -Doc/library/asyncio-eventloop.rst Doc/library/asyncio-extending.rst -Doc/library/asyncio-future.rst Doc/library/asyncio-policy.rst -Doc/library/asyncio-stream.rst Doc/library/asyncio-subprocess.rst Doc/library/asyncio-task.rst -Doc/library/audioop.rst Doc/library/bdb.rst Doc/library/bisect.rst -Doc/library/bz2.rst Doc/library/calendar.rst -Doc/library/cgi.rst -Doc/library/chunk.rst -Doc/library/cmath.rst Doc/library/cmd.rst -Doc/library/code.rst -Doc/library/codecs.rst -Doc/library/codeop.rst Doc/library/collections.abc.rst Doc/library/collections.rst -Doc/library/compileall.rst Doc/library/concurrent.futures.rst -Doc/library/concurrent.rst Doc/library/configparser.rst -Doc/library/constants.rst -Doc/library/contextlib.rst -Doc/library/copy.rst Doc/library/csv.rst -Doc/library/ctypes.rst -Doc/library/curses.ascii.rst -Doc/library/curses.rst Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst -Doc/library/devmode.rst -Doc/library/difflib.rst -Doc/library/dis.rst -Doc/library/doctest.rst Doc/library/email.charset.rst Doc/library/email.compat32-message.rst -Doc/library/email.encoders.rst Doc/library/email.errors.rst -Doc/library/email.generator.rst -Doc/library/email.headerregistry.rst -Doc/library/email.message.rst Doc/library/email.mime.rst Doc/library/email.parser.rst Doc/library/email.policy.rst @@ -132,150 +53,74 @@ Doc/library/enum.rst Doc/library/exceptions.rst Doc/library/faulthandler.rst Doc/library/fcntl.rst -Doc/library/filecmp.rst -Doc/library/fileinput.rst -Doc/library/fractions.rst Doc/library/ftplib.rst -Doc/library/functions.rst Doc/library/functools.rst -Doc/library/getopt.rst -Doc/library/getpass.rst -Doc/library/gettext.rst -Doc/library/graphlib.rst -Doc/library/gzip.rst -Doc/library/hashlib.rst Doc/library/http.client.rst Doc/library/http.cookiejar.rst Doc/library/http.cookies.rst Doc/library/http.server.rst -Doc/library/idle.rst -Doc/library/importlib.resources.abc.rst -Doc/library/importlib.resources.rst Doc/library/importlib.rst Doc/library/inspect.rst -Doc/library/io.rst -Doc/library/json.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst -Doc/library/logging.rst Doc/library/lzma.rst Doc/library/mailbox.rst Doc/library/mmap.rst -Doc/library/msilib.rst -Doc/library/msvcrt.rst Doc/library/multiprocessing.rst Doc/library/multiprocessing.shared_memory.rst -Doc/library/netrc.rst -Doc/library/nntplib.rst Doc/library/numbers.rst -Doc/library/operator.rst Doc/library/optparse.rst -Doc/library/os.path.rst Doc/library/os.rst -Doc/library/ossaudiodev.rst Doc/library/pickle.rst Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst -Doc/library/poplib.rst -Doc/library/posix.rst -Doc/library/pprint.rst Doc/library/profile.rst -Doc/library/pty.rst -Doc/library/py_compile.rst Doc/library/pyclbr.rst Doc/library/pydoc.rst Doc/library/pyexpat.rst Doc/library/random.rst -Doc/library/re.rst Doc/library/readline.rst -Doc/library/reprlib.rst Doc/library/resource.rst Doc/library/rlcompleter.rst -Doc/library/sched.rst Doc/library/select.rst -Doc/library/selectors.rst -Doc/library/shelve.rst -Doc/library/shutil.rst Doc/library/signal.rst -Doc/library/site.rst Doc/library/smtplib.rst Doc/library/socket.rst -Doc/library/socketserver.rst Doc/library/ssl.rst -Doc/library/stat.rst Doc/library/stdtypes.rst Doc/library/string.rst -Doc/library/struct.rst Doc/library/subprocess.rst -Doc/library/sunau.rst -Doc/library/sys.rst -Doc/library/sys_path_init.rst -Doc/library/sysconfig.rst Doc/library/syslog.rst Doc/library/tarfile.rst -Doc/library/telnetlib.rst Doc/library/tempfile.rst Doc/library/termios.rst Doc/library/test.rst -Doc/library/textwrap.rst -Doc/library/threading.rst -Doc/library/time.rst Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst -Doc/library/tkinter.tix.rst Doc/library/tkinter.ttk.rst Doc/library/traceback.rst -Doc/library/tty.rst -Doc/library/turtle.rst -Doc/library/unittest.mock-examples.rst Doc/library/unittest.mock.rst Doc/library/unittest.rst -Doc/library/urllib.error.rst Doc/library/urllib.parse.rst Doc/library/urllib.request.rst -Doc/library/uuid.rst -Doc/library/wave.rst -Doc/library/weakref.rst -Doc/library/webbrowser.rst -Doc/library/winreg.rst -Doc/library/winsound.rst Doc/library/wsgiref.rst -Doc/library/xdrlib.rst Doc/library/xml.dom.minidom.rst Doc/library/xml.dom.pulldom.rst Doc/library/xml.dom.rst -Doc/library/xml.etree.elementtree.rst -Doc/library/xml.rst Doc/library/xml.sax.handler.rst Doc/library/xml.sax.reader.rst Doc/library/xml.sax.rst -Doc/library/xml.sax.utils.rst Doc/library/xmlrpc.client.rst -Doc/library/xmlrpc.rst Doc/library/xmlrpc.server.rst Doc/library/zlib.rst -Doc/license.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/reference/expressions.rst Doc/reference/import.rst -Doc/reference/lexical_analysis.rst Doc/reference/simple_stmts.rst -Doc/tutorial/appendix.rst -Doc/tutorial/classes.rst -Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst -Doc/tutorial/errors.rst -Doc/tutorial/inputoutput.rst -Doc/tutorial/interactive.rst -Doc/tutorial/introduction.rst -Doc/tutorial/modules.rst -Doc/tutorial/stdlib2.rst -Doc/using/cmdline.rst -Doc/using/configure.rst -Doc/using/unix.rst Doc/using/windows.rst Doc/whatsnew/2.0.rst Doc/whatsnew/2.1.rst diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py new file mode 100644 index 00000000000000..809a8d63087e12 --- /dev/null +++ b/Doc/tools/check-warnings.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +""" +Check the output of running Sphinx in nit-picky mode (missing references). +""" +from __future__ import annotations + +import argparse +import itertools +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import TextIO + +# Exclude these whether they're dirty or clean, +# because they trigger a rebuild of dirty files. +EXCLUDE_FILES = { + "Doc/whatsnew/changelog.rst", +} + +# Subdirectories of Doc/ to exclude. +EXCLUDE_SUBDIRS = { + ".env", + ".venv", + "env", + "includes", + "venv", +} + +# Regex pattern to match the parts of a Sphinx warning +WARNING_PATTERN = re.compile( + r"(?P([A-Za-z]:[\\/])?[^:]+):(?P\d+): WARNING: (?P.+)" +) + +# Regex pattern to match the line numbers in a Git unified diff +DIFF_PATTERN = re.compile( + r"^@@ -(?P\d+)(?:,(?P\d+))? \+(?P\d+)(?:,(?P\d+))? @@", + flags=re.MULTILINE, +) + + +def get_diff_files(ref_a: str, ref_b: str, filter_mode: str = "") -> set[Path]: + """List the files changed between two Git refs, filtered by change type.""" + added_files_result = subprocess.run( + [ + "git", + "diff", + f"--diff-filter={filter_mode}", + "--name-only", + f"{ref_a}...{ref_b}", + "--", + ], + stdout=subprocess.PIPE, + check=True, + text=True, + encoding="UTF-8", + ) + + added_files = added_files_result.stdout.strip().split("\n") + return {Path(file.strip()) for file in added_files if file.strip()} + + +def get_diff_lines(ref_a: str, ref_b: str, file: Path) -> list[int]: + """List the lines changed between two Git refs for a specific file.""" + diff_output = subprocess.run( + [ + "git", + "diff", + "--unified=0", + f"{ref_a}...{ref_b}", + "--", + str(file), + ], + stdout=subprocess.PIPE, + check=True, + text=True, + encoding="UTF-8", + ) + + # Scrape line offsets + lengths from diff and convert to line numbers + line_matches = DIFF_PATTERN.finditer(diff_output.stdout) + # Removed and added line counts are 1 if not printed + line_match_values = [ + line_match.groupdict(default=1) for line_match in line_matches + ] + line_ints = [ + (int(match_value["lineb"]), int(match_value["added"])) + for match_value in line_match_values + ] + line_ranges = [ + range(line_b, line_b + added) for line_b, added in line_ints + ] + line_numbers = list(itertools.chain(*line_ranges)) + + return line_numbers + + +def get_para_line_numbers(file_obj: TextIO) -> list[list[int]]: + """Get the line numbers of text in a file object, grouped by paragraph.""" + paragraphs = [] + prev_line = None + for lineno, line in enumerate(file_obj): + lineno = lineno + 1 + if prev_line is None or (line.strip() and not prev_line.strip()): + paragraph = [lineno - 1] + paragraphs.append(paragraph) + paragraph.append(lineno) + prev_line = line + return paragraphs + + +def filter_and_parse_warnings( + warnings: list[str], files: set[Path] +) -> list[re.Match[str]]: + """Get the warnings matching passed files and parse them with regex.""" + filtered_warnings = [ + warning + for warning in warnings + if any(str(file) in warning for file in files) + ] + warning_matches = [ + WARNING_PATTERN.fullmatch(warning.strip()) + for warning in filtered_warnings + ] + non_null_matches = [warning for warning in warning_matches if warning] + return non_null_matches + + +def filter_warnings_by_diff( + warnings: list[re.Match[str]], ref_a: str, ref_b: str, file: Path +) -> list[re.Match[str]]: + """Filter the passed per-file warnings to just those on changed lines.""" + diff_lines = get_diff_lines(ref_a, ref_b, file) + with file.open(encoding="UTF-8") as file_obj: + paragraphs = get_para_line_numbers(file_obj) + touched_paras = [ + para_lines + for para_lines in paragraphs + if set(diff_lines) & set(para_lines) + ] + touched_para_lines = set(itertools.chain(*touched_paras)) + warnings_infile = [ + warning for warning in warnings if str(file) in warning["file"] + ] + warnings_touched = [ + warning + for warning in warnings_infile + if int(warning["line"]) in touched_para_lines + ] + return warnings_touched + + +def process_touched_warnings( + warnings: list[str], ref_a: str, ref_b: str +) -> list[re.Match[str]]: + """Filter a list of Sphinx warnings to those affecting touched lines.""" + added_files, modified_files = tuple( + get_diff_files(ref_a, ref_b, filter_mode=mode) for mode in ("A", "M") + ) + + warnings_added = filter_and_parse_warnings(warnings, added_files) + warnings_modified = filter_and_parse_warnings(warnings, modified_files) + + modified_files_warned = { + file + for file in modified_files + if any(str(file) in warning["file"] for warning in warnings_modified) + } + + warnings_modified_touched = [ + filter_warnings_by_diff(warnings_modified, ref_a, ref_b, file) + for file in modified_files_warned + ] + warnings_touched = warnings_added + list( + itertools.chain(*warnings_modified_touched) + ) + + return warnings_touched + + +def annotate_diff( + warnings: list[str], ref_a: str = "main", ref_b: str = "HEAD" +) -> None: + """ + Convert Sphinx warning messages to GitHub Actions for changed paragraphs. + + Converts lines like: + .../Doc/library/cgi.rst:98: WARNING: reference target not found + to: + ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found + + See: + https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message + """ + warnings_touched = process_touched_warnings(warnings, ref_a, ref_b) + print("Emitting doc warnings matching modified lines:") + for warning in warnings_touched: + print("::warning file={file},line={line}::{msg}".format_map(warning)) + print(warning[0]) + if not warnings_touched: + print("None") + + +def fail_if_regression( + warnings: list[str], files_with_expected_nits: set[str], files_with_nits: set[str] +) -> int: + """ + Ensure some files always pass Sphinx nit-picky mode (no missing references). + These are files which are *not* in .nitignore. + """ + all_rst = { + str(rst) + for rst in Path("Doc/").rglob("*.rst") + if rst.parts[1] not in EXCLUDE_SUBDIRS + } + should_be_clean = all_rst - files_with_expected_nits - EXCLUDE_FILES + problem_files = sorted(should_be_clean & files_with_nits) + if problem_files: + print("\nError: must not contain warnings:\n") + for filename in problem_files: + print(filename) + for warning in warnings: + if filename in warning: + if match := WARNING_PATTERN.fullmatch(warning): + print(" {line}: {msg}".format_map(match)) + return -1 + return 0 + + +def fail_if_improved( + files_with_expected_nits: set[str], files_with_nits: set[str] +) -> int: + """ + We may have fixed warnings in some files so that the files are now completely clean. + Good news! Let's add them to .nitignore to prevent regression. + """ + files_with_no_nits = files_with_expected_nits - files_with_nits + if files_with_no_nits: + print("\nCongratulations! You improved:\n") + for filename in sorted(files_with_no_nits): + print(filename) + print("\nPlease remove from Doc/tools/.nitignore\n") + return -1 + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--annotate-diff", + nargs="*", + metavar=("BASE_REF", "HEAD_REF"), + help="Add GitHub Actions annotations on the diff for warnings on " + "lines changed between the given refs (main and HEAD, by default)", + ) + parser.add_argument( + "--fail-if-regression", + action="store_true", + help="Fail if known-good files have warnings", + ) + parser.add_argument( + "--fail-if-improved", + action="store_true", + help="Fail if new files with no nits are found", + ) + + args = parser.parse_args(argv) + if args.annotate_diff is not None and len(args.annotate_diff) > 2: + parser.error( + "--annotate-diff takes between 0 and 2 ref args, not " + f"{len(args.annotate_diff)} {tuple(args.annotate_diff)}" + ) + exit_code = 0 + + wrong_directory_msg = "Must run this script from the repo root" + assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg + + with Path("Doc/sphinx-warnings.txt").open(encoding="UTF-8") as f: + warnings = f.read().splitlines() + + cwd = str(Path.cwd()) + os.path.sep + files_with_nits = { + warning.removeprefix(cwd).split(":")[0] + for warning in warnings + if "Doc/" in warning + } + + with Path("Doc/tools/.nitignore").open(encoding="UTF-8") as clean_files: + files_with_expected_nits = { + filename.strip() + for filename in clean_files + if filename.strip() and not filename.startswith("#") + } + + if args.annotate_diff is not None: + annotate_diff(warnings, *args.annotate_diff) + + if args.fail_if_regression: + exit_code += fail_if_regression( + warnings, files_with_expected_nits, files_with_nits + ) + + if args.fail_if_improved: + exit_code += fail_if_improved(files_with_expected_nits, files_with_nits) + + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index cd8d9febb0d13b..31c2544caf601c 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -14,29 +14,27 @@ from os import getenv, path from time import asctime from pprint import pformat + +from docutils import nodes, utils from docutils.io import StringOutput from docutils.parsers.rst import Directive from docutils.utils import new_document - -from docutils import nodes, utils - from sphinx import addnodes from sphinx.builders import Builder -try: - from sphinx.errors import NoUri -except ImportError: - from sphinx.environment import NoUri +from sphinx.domains.python import PyFunction, PyMethod +from sphinx.errors import NoUri from sphinx.locale import _ as sphinx_gettext -from sphinx.util import status_iterator, logging +from sphinx.util import logging from sphinx.util.docutils import SphinxDirective from sphinx.util.nodes import split_explicit_title from sphinx.writers.text import TextWriter, TextTranslator try: - from sphinx.domains.python import PyFunction, PyMethod + # Sphinx 6+ + from sphinx.util.display import status_iterator except ImportError: - from sphinx.domains.python import PyClassmember as PyMethod - from sphinx.domains.python import PyModulelevel as PyFunction + # Deprecated in Sphinx 6.1, will be removed in Sphinx 8 + from sphinx.util import status_iterator ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' @@ -100,14 +98,13 @@ class ImplementationDetail(Directive): final_argument_whitespace = True # This text is copied to templates/dummy.html - label_text = 'CPython implementation detail:' + label_text = sphinx_gettext('CPython implementation detail:') def run(self): self.assert_has_content() pnode = nodes.compound(classes=['impl-detail']) - label = sphinx_gettext(self.label_text) content = self.content - add_text = nodes.strong(label, label) + add_text = nodes.strong(self.label_text, self.label_text) self.state.nested_parse(content, self.content_offset, pnode) content = nodes.inline(pnode[0].rawsource, translatable=True) content.source = pnode[0].source @@ -130,8 +127,8 @@ class Availability(SphinxDirective): # known platform, libc, and threading implementations known_platforms = frozenset({ "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", - "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", "Unix", "VxWorks", - "WASI", "Windows", "macOS", + "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", + "Unix", "VxWorks", "WASI", "Windows", "macOS", # libc "BSD libc", "glibc", "musl", # POSIX platforms with pthreads @@ -182,7 +179,7 @@ def parse_platforms(self): if unknown: cls = type(self) logger = logging.getLogger(cls.__qualname__) - logger.warn( + logger.warning( f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " f"in '.. availability:: {self.arguments[0]}', see " f"{__file__}:{cls.__qualname__}.known_platforms for a set " @@ -236,9 +233,9 @@ class AuditEvent(Directive): final_argument_whitespace = True _label = [ - "Raises an :ref:`auditing event ` {name} with no arguments.", - "Raises an :ref:`auditing event ` {name} with argument {args}.", - "Raises an :ref:`auditing event ` {name} with arguments {args}.", + sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), ] @property @@ -254,7 +251,7 @@ def run(self): else: args = [] - label = sphinx_gettext(self._label[min(2, len(args))]) + label = self._label[min(2, len(args))] text = label.format(name="``{}``".format(name), args=", ".join("``{}``".format(a) for a in args if a)) @@ -269,7 +266,7 @@ def run(self): info = env.all_audit_events.setdefault(name, new_info) if info is not new_info: if not self._do_args_match(info['args'], new_info['args']): - self.logger.warn( + self.logger.warning( "Mismatched arguments for audit-event {}: {!r} != {!r}" .format(name, info['args'], new_info['args']) ) @@ -416,8 +413,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' - _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' + _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') + _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') def run(self): node = addnodes.versionmodified() @@ -433,7 +430,6 @@ def run(self): else: label = self._removed_label - label = sphinx_gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], @@ -546,7 +542,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.env.logger.warn('label %r not in documentation' % label) + self.env.logger.warning(f'label {label!r} not in documentation') continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) @@ -561,6 +557,7 @@ def finish(self): try: f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8')) f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8')) + f.write('# as part of the release process.\n'.encode('utf-8')) f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8')) finally: f.close() @@ -610,6 +607,13 @@ def parse_pdb_command(env, sig, signode): return fullname +def parse_monitoring_event(env, sig, signode): + """Transform a monitoring event signature into RST nodes.""" + signode += addnodes.desc_addname('sys.monitoring.events.', 'sys.monitoring.events.') + signode += addnodes.desc_name(sig, sig) + return sig + + def process_audit_events(app, doctree, fromdocname): for node in doctree.traverse(audit_event_list): break @@ -710,7 +714,7 @@ def setup(app): app.add_builder(PydocTopicsBuilder) app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) - app.add_object_type('2to3fixer', '2to3fixer', '%s (2to3 fixer)') + app.add_object_type('monitoring-event', 'monitoring-event', '%s (monitoring event)', parse_monitoring_event) app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 7920e0619f9337..b5353d6fb77ab4 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -49,12 +49,12 @@

Unpacking

Unix users should download the .tar.bz2 archives; these are bzipped tar archives and can be handled in the usual way using tar and the bzip2 -program. The InfoZIP unzip program can be +program. The Info-ZIP unzip program can be used to handle the ZIP archives if desired. The .tar.bz2 archives provide the best compression and fastest download times.

Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the InfoZIP zip program.

+platform. These are created on Unix using the Info-ZIP zip program.

Problems

diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index a96746b69fd41b..1e3ab7cfe02fee 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -62,6 +62,7 @@

{{ docstitle|e }}

+ {% endblock %} diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index b91f8138553e62..9498b2ccc5af92 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -4,16 +4,16 @@ {%- if outdated %}
{% trans %}This document is for an old version of Python that is no longer supported. - You should upgrade, and read the {% endtrans %} - {% trans %} Python documentation for the current stable release{% endtrans %}. + You should upgrade, and read the{% endtrans %} + {% trans %}Python documentation for the current stable release{% endtrans %}.
{%- endif %} {%- if is_deployment_preview %}
{% trans %}This is a deploy preview created from a pull request. - For authoritative documentation, see the {% endtrans %} - {% trans %} the current stable release{% endtrans %}. + For authoritative documentation, see{% endtrans %} + {% trans %}the current stable release{% endtrans %}.
{%- endif %} {% endblock %} diff --git a/Doc/tools/templates/search.html b/Doc/tools/templates/search.html index f2ac2ea0f09873..852974461380f2 100644 --- a/Doc/tools/templates/search.html +++ b/Doc/tools/templates/search.html @@ -1,48 +1,62 @@ {% extends "!search.html" %} {% block extrahead %} {{ super() }} + -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/Doc/tools/touch-clean-files.py b/Doc/tools/touch-clean-files.py deleted file mode 100644 index 19bc1be31deb26..00000000000000 --- a/Doc/tools/touch-clean-files.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -""" -Touch files that must pass Sphinx nit-picky mode -so they are rebuilt and we can catch regressions. -""" - -from pathlib import Path - -wrong_directory_msg = "Must run this script from the repo root" -assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg - -# Exclude these whether they're dirty or clean, -# because they trigger a rebuild of dirty files. -EXCLUDE_FILES = { - Path("Doc/whatsnew/changelog.rst"), -} - -# Subdirectories of Doc/ to exclude. -EXCLUDE_SUBDIRS = { - ".env", - ".venv", - "env", - "includes", - "venv", -} - -ALL_RST = { - rst for rst in Path("Doc/").rglob("*.rst") if rst.parts[1] not in EXCLUDE_SUBDIRS -} - -with Path("Doc/tools/.nitignore").open() as clean_files: - DIRTY = { - Path(filename.strip()) - for filename in clean_files - if filename.strip() and not filename.startswith("#") - } - -CLEAN = ALL_RST - DIRTY - EXCLUDE_FILES - -print("Touching:") -for filename in sorted(CLEAN): - print(filename) - filename.touch() -print(f"Touched {len(CLEAN)} files") diff --git a/Doc/tools/warnings-to-gh-actions.py b/Doc/tools/warnings-to-gh-actions.py deleted file mode 100644 index da33a4ede07abc..00000000000000 --- a/Doc/tools/warnings-to-gh-actions.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -""" -Convert Sphinx warning messages to GitHub Actions. - -Converts lines like: - .../Doc/library/cgi.rst:98: WARNING: reference target not found -to: - ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found - -Non-matching lines are echoed unchanged. - -see: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message -""" - -import re -import sys - -pattern = re.compile(r'(?P[^:]+):(?P\d+): WARNING: (?P.+)') - -for line in sys.stdin: - if match := pattern.fullmatch(line.strip()): - print('::warning file={file},line={line}::{msg}'.format_map(match)) - else: - print(line) diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 241a812037469e..588591fcdb726f 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -101,8 +101,8 @@ in the script:: The Customization Modules ------------------------- -Python provides two hooks to let you customize it: :mod:`sitecustomize` and -:mod:`usercustomize`. To see how it works, you need first to find the location +Python provides two hooks to let you customize it: :index:`sitecustomize` and +:index:`usercustomize`. To see how it works, you need first to find the location of your user site-packages directory. Start Python and run this code:: >>> import site @@ -113,9 +113,9 @@ Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless it is started with the :option:`-s` option to disable the automatic import. -:mod:`sitecustomize` works in the same way, but is typically created by an +:index:`sitecustomize` works in the same way, but is typically created by an administrator of the computer in the global site-packages directory, and is -imported before :mod:`usercustomize`. See the documentation of the :mod:`site` +imported before :index:`usercustomize`. See the documentation of the :mod:`site` module for more details. diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 06445e000c1ef6..3bf138ca225ee5 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -91,7 +91,7 @@ Attributes may be read-only or writable. In the latter case, assignment to attributes is possible. Module attributes are writable: you can write ``modname.the_answer = 42``. Writable attributes may also be deleted with the :keyword:`del` statement. For example, ``del modname.the_answer`` will remove -the attribute :attr:`the_answer` from the object named by ``modname``. +the attribute :attr:`!the_answer` from the object named by ``modname``. Namespaces are created at different moments and have different lifetimes. The namespace containing the built-in names is created when the Python interpreter @@ -249,7 +249,7 @@ created. This is basically a wrapper around the contents of the namespace created by the class definition; we'll learn more about class objects in the next section. The original local scope (the one in effect just before the class definition was entered) is reinstated, and the class object is bound here to the -class name given in the class definition header (:class:`ClassName` in the +class name given in the class definition header (:class:`!ClassName` in the example). @@ -276,7 +276,7 @@ definition looked like this:: then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning an integer and a function object, respectively. Class attributes can also be assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`__doc__` is also a valid attribute, returning the docstring belonging to +:attr:`!__doc__` is also a valid attribute, returning the docstring belonging to the class: ``"A simple example class"``. Class *instantiation* uses function notation. Just pretend that the class @@ -291,20 +291,20 @@ variable ``x``. The instantiation operation ("calling" a class object) creates an empty object. Many classes like to create objects with instances customized to a specific initial state. Therefore a class may define a special method named -:meth:`__init__`, like this:: +:meth:`~object.__init__`, like this:: def __init__(self): self.data = [] -When a class defines an :meth:`__init__` method, class instantiation -automatically invokes :meth:`__init__` for the newly created class instance. So +When a class defines an :meth:`~object.__init__` method, class instantiation +automatically invokes :meth:`!__init__` for the newly created class instance. So in this example, a new, initialized instance can be obtained by:: x = MyClass() -Of course, the :meth:`__init__` method may have arguments for greater +Of course, the :meth:`~object.__init__` method may have arguments for greater flexibility. In that case, arguments given to the class instantiation operator -are passed on to :meth:`__init__`. For example, :: +are passed on to :meth:`!__init__`. For example, :: >>> class Complex: ... def __init__(self, realpart, imagpart): @@ -328,7 +328,7 @@ attribute names: data attributes and methods. *data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. For example, if -``x`` is the instance of :class:`MyClass` created above, the following piece of +``x`` is the instance of :class:`!MyClass` created above, the following piece of code will print the value ``16``, without leaving a trace:: x.counter = 1 @@ -363,7 +363,7 @@ Usually, a method is called right after it is bound:: x.f() -In the :class:`MyClass` example, this will return the string ``'hello world'``. +In the :class:`!MyClass` example, this will return the string ``'hello world'``. However, it is not necessary to call a method right away: ``x.f`` is a method object, and can be stored away and called at a later time. For example:: @@ -375,7 +375,7 @@ will continue to print ``hello world`` until the end of time. What exactly happens when a method is called? You may have noticed that ``x.f()`` was called without an argument above, even though the function -definition for :meth:`f` specified an argument. What happened to the argument? +definition for :meth:`!f` specified an argument. What happened to the argument? Surely Python raises an exception when a function that requires an argument is called without any --- even if the argument isn't actually used... @@ -532,9 +532,9 @@ variable in the class is also ok. For example:: h = g -Now ``f``, ``g`` and ``h`` are all attributes of class :class:`C` that refer to +Now ``f``, ``g`` and ``h`` are all attributes of class :class:`!C` that refer to function objects, and consequently they are all methods of instances of -:class:`C` --- ``h`` being exactly equivalent to ``g``. Note that this practice +:class:`!C` --- ``h`` being exactly equivalent to ``g``. Note that this practice usually only serves to confuse the reader of a program. Methods may call other methods by using method attributes of the ``self`` @@ -581,7 +581,7 @@ this:: . -The name :class:`BaseClassName` must be defined in a +The name :class:`!BaseClassName` must be defined in a namespace accessible from the scope containing the derived class definition. In place of a base class name, other arbitrary expressions are also allowed. This can be useful, for example, when the base @@ -645,9 +645,9 @@ multiple base classes looks like this:: For most purposes, in the simplest cases, you can think of the search for attributes inherited from a parent class as depth-first, left-to-right, not searching twice in the same class where there is an overlap in the hierarchy. -Thus, if an attribute is not found in :class:`DerivedClassName`, it is searched -for in :class:`Base1`, then (recursively) in the base classes of :class:`Base1`, -and if it was not found there, it was searched for in :class:`Base2`, and so on. +Thus, if an attribute is not found in :class:`!DerivedClassName`, it is searched +for in :class:`!Base1`, then (recursively) in the base classes of :class:`!Base1`, +and if it was not found there, it was searched for in :class:`!Base2`, and so on. In fact, it is slightly more complex than that; the method resolution order changes dynamically to support cooperative calls to :func:`super`. This @@ -760,7 +760,8 @@ is to use :mod:`dataclasses` for this purpose:: A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`!readline` that get the +can define a class with methods :meth:`~io.TextIOBase.read` and +:meth:`~io.TextIOBase.readline` that get the data from a string buffer instead, and pass it as an argument. .. (Unfortunately, this technique has its limitations: a class can't define @@ -768,8 +769,10 @@ data from a string buffer instead, and pass it as an argument. or arithmetic operators, and assigning such a "pseudo-file" to sys.stdin will not cause the interpreter to read further input from it.) -Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`m`, and ``m.__func__`` is the function object +:ref:`Instance method objects ` have attributes, too: +:attr:`m.__self__ ` is the instance +object with the method :meth:`!m`, and :attr:`m.__func__ ` is +the :ref:`function object ` corresponding to the method. @@ -818,9 +821,9 @@ using the :func:`next` built-in function; this example shows how it all works:: StopIteration Having seen the mechanics behind the iterator protocol, it is easy to add -iterator behavior to your classes. Define an :meth:`__iter__` method which +iterator behavior to your classes. Define an :meth:`~container.__iter__` method which returns an object with a :meth:`~iterator.__next__` method. If the class -defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: +defines :meth:`!__next__`, then :meth:`!__iter__` can just return ``self``:: class Reverse: """Iterator for looping over a sequence backwards.""" @@ -879,7 +882,7 @@ easy to create:: Anything that can be done with generators can also be done with class-based iterators as described in the previous section. What makes generators so -compact is that the :meth:`__iter__` and :meth:`~generator.__next__` methods +compact is that the :meth:`~iterator.__iter__` and :meth:`~generator.__next__` methods are created automatically. Another key feature is that the local variables and execution state are diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index c9b3d982c31c9a..aa9caa101da40a 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -4,8 +4,8 @@ More Control Flow Tools *********************** -Besides the :keyword:`while` statement just introduced, Python uses the usual -flow control statements known from other languages, with some twists. +As well as the :keyword:`while` statement just introduced, Python uses a few more +that we will encounter in this chapter. .. _tut-if: @@ -163,14 +163,21 @@ arguments. In chapter :ref:`tut-structures`, we will discuss in more detail abo :keyword:`!break` and :keyword:`!continue` Statements, and :keyword:`!else` Clauses on Loops ============================================================================================ -The :keyword:`break` statement, like in C, breaks out of the innermost enclosing +The :keyword:`break` statement breaks out of the innermost enclosing :keyword:`for` or :keyword:`while` loop. -Loop statements may have an :keyword:`!else` clause; it is executed when the loop -terminates through exhaustion of the iterable (with :keyword:`for`) or when the -condition becomes false (with :keyword:`while`), but not when the loop is -terminated by a :keyword:`break` statement. This is exemplified by the -following loop, which searches for prime numbers:: +A :keyword:`!for` or :keyword:`!while` loop can include an :keyword:`!else` clause. + +In a :keyword:`for` loop, the :keyword:`!else` clause is executed +after the loop reaches its final iteration. + +In a :keyword:`while` loop, it's executed after the loop's condition becomes false. + +In either kind of loop, the :keyword:`!else` clause is **not** executed +if the loop was terminated by a :keyword:`break`. + +This is exemplified in the following :keyword:`!for` loop, +which searches for prime numbers:: >>> for n in range(2, 10): ... for x in range(2, n): @@ -307,8 +314,9 @@ you can use the class name followed by an argument list resembling a constructor, but with the ability to capture attributes into variables:: class Point: - x: int - y: int + def __init__(self, x, y): + self.x = x + self.y = y def where_is(point): match point: @@ -342,7 +350,13 @@ Dotted names (like ``foo.bar``), attribute names (the ``x=`` and ``y=`` above) o (recognized by the "(...)" next to them like ``Point`` above) are never assigned to. Patterns can be arbitrarily nested. For example, if we have a short -list of points, we could match it like this:: +list of Points, with ``__match_args__`` added, we could match it like this:: + + class Point: + __match_args__ = ('x', 'y') + def __init__(self, x, y): + self.x = x + self.y = y match points: case []: @@ -520,7 +534,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`append` shown in the example is defined for list objects; it + The method :meth:`!append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. @@ -1032,7 +1046,7 @@ Function Annotations information about the types used by user-defined functions (see :pep:`3107` and :pep:`484` for more information). -:term:`Annotations ` are stored in the :attr:`__annotations__` +:term:`Annotations ` are stored in the :attr:`!__annotations__` attribute of the function as a dictionary and have no effect on any other part of the function. Parameter annotations are defined by a colon after the parameter name, followed by an expression evaluating to the value of the annotation. Return annotations are diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index c8e89d9b79bddd..87614d082a1d4e 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -143,8 +143,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`append`. To retrieve an item from the -top of the stack, use :meth:`pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the +top of the stack, use :meth:`~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -341,7 +341,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`pop` method +value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -501,8 +501,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`append` and -:meth:`extend`. +assignments, slice assignments, or methods like :meth:`~list.append` and +:meth:`~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of @@ -567,7 +567,7 @@ Looping Techniques ================== When looping through dictionaries, the key and corresponding value can be -retrieved at the same time using the :meth:`items` method. :: +retrieved at the same time using the :meth:`~dict.items` method. :: >>> knights = {'gallahad': 'the pure', 'robin': 'the brave'} >>> for k, v in knights.items(): diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index ca5dc3314c63b6..4058ebe8efdb42 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -108,8 +108,7 @@ The :keyword:`try` statement works as follows. * If an exception occurs which does not match the exception named in the *except clause*, it is passed on to outer :keyword:`try` statements; if no handler is - found, it is an *unhandled exception* and execution stops with a message as - shown above. + found, it is an *unhandled exception* and execution stops with an error message. A :keyword:`try` statement may have more than one *except clause*, to specify handlers for different exceptions. At most one handler will be executed. @@ -154,7 +153,7 @@ exception type. The *except clause* may specify a variable after the exception name. The variable is bound to the exception instance which typically has an ``args`` attribute that stores the arguments. For convenience, builtin exception -types define :meth:`__str__` to print all the arguments without explicitly +types define :meth:`~object.__str__` to print all the arguments without explicitly accessing ``.args``. :: >>> try: @@ -174,7 +173,7 @@ accessing ``.args``. :: x = spam y = eggs -The exception's :meth:`__str__` output is printed as the last part ('detail') +The exception's :meth:`~object.__str__` output is printed as the last part ('detail') of the message for unhandled exceptions. :exc:`BaseException` is the common base class of all exceptions. One of its @@ -535,11 +534,20 @@ of a certain type while letting all other exceptions propagate to other clauses and eventually to be reraised. :: >>> def f(): - ... raise ExceptionGroup("group1", - ... [OSError(1), - ... SystemError(2), - ... ExceptionGroup("group2", - ... [OSError(3), RecursionError(4)])]) + ... raise ExceptionGroup( + ... "group1", + ... [ + ... OSError(1), + ... SystemError(2), + ... ExceptionGroup( + ... "group2", + ... [ + ... OSError(3), + ... RecursionError(4) + ... ] + ... ) + ... ] + ... ) ... >>> try: ... f() @@ -578,6 +586,8 @@ the following pattern:: ... +.. _tut-exception-notes: + Enriching Exceptions with Notes =============================== diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index 306b1eba3c45b8..0795e2fef98830 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -137,7 +137,7 @@ the :func:`math.isclose` function can be useful for comparing inexact values: True Alternatively, the :func:`round` function can be used to compare rough -approximations:: +approximations: .. doctest:: @@ -148,9 +148,9 @@ Binary floating-point arithmetic holds many surprises like this. The problem with "0.1" is explained in precise detail below, in the "Representation Error" section. See `Examples of Floating Point Problems `_ for -a pleasant summary of how binary floating point works and the kinds of +a pleasant summary of how binary floating-point works and the kinds of problems commonly encountered in practice. Also see -`The Perils of Floating Point `_ +`The Perils of Floating Point `_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly @@ -174,7 +174,7 @@ Another form of exact arithmetic is supported by the :mod:`fractions` module which implements arithmetic based on rational numbers (so the numbers like 1/3 can be represented exactly). -If you are a heavy user of floating point operations you should take a look +If you are a heavy user of floating-point operations you should take a look at the NumPy package and many other packages for mathematical and statistical operations supplied by the SciPy project. See . @@ -268,12 +268,14 @@ decimal fractions cannot be represented exactly as binary (base 2) fractions. This is the chief reason why Python (or Perl, C, C++, Java, Fortran, and many others) often won't display the exact decimal number you expect. -Why is that? 1/10 is not exactly representable as a binary fraction. Almost all -machines today (November 2000) use IEEE-754 floating point arithmetic, and -almost all platforms map Python floats to IEEE-754 "double precision". 754 -doubles contain 53 bits of precision, so on input the computer strives to -convert 0.1 to the closest fraction it can of the form *J*/2**\ *N* where *J* is -an integer containing exactly 53 bits. Rewriting :: +Why is that? 1/10 is not exactly representable as a binary fraction. Since at +least 2000, almost all machines use IEEE 754 binary floating-point arithmetic, +and almost all platforms map Python floats to IEEE 754 binary64 "double +precision" values. IEEE 754 binary64 values contain 53 bits of precision, so +on input the computer strives to convert 0.1 to the closest fraction it can of +the form *J*/2**\ *N* where *J* is an integer containing exactly 53 bits. +Rewriting +:: 1 / 10 ~= J / (2**N) @@ -308,7 +310,8 @@ by rounding up: >>> q+1 7205759403792794 -Therefore the best possible approximation to 1/10 in 754 double precision is:: +Therefore the best possible approximation to 1/10 in IEEE 754 double precision +is:: 7205759403792794 / 2 ** 56 @@ -321,7 +324,7 @@ if we had not rounded up, the quotient would have been a little bit smaller than 1/10. But in no case can it be *exactly* 1/10! So the computer never "sees" 1/10: what it sees is the exact fraction given -above, the best 754 double approximation it can get: +above, the best IEEE 754 double approximation it can get: .. doctest:: diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index f5cdd84cbadefe..fe9ca9ccb9c7e0 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -15,7 +15,7 @@ Fancier Output Formatting ========================= So far we've encountered two ways of writing values: *expression statements* and -the :func:`print` function. (A third way is using the :meth:`write` method +the :func:`print` function. (A third way is using the :meth:`~io.TextIOBase.write` method of file objects; the standard output file can be referenced as ``sys.stdout``. See the Library Reference for more information on this.) @@ -456,8 +456,8 @@ to the very file end with ``seek(0, 2)``) and the only valid *offset* values are those returned from the ``f.tell()``, or zero. Any other *offset* value produces undefined behaviour. -File objects have some additional methods, such as :meth:`~file.isatty` and -:meth:`~file.truncate` which are less frequently used; consult the Library +File objects have some additional methods, such as :meth:`~io.IOBase.isatty` and +:meth:`~io.IOBase.truncate` which are less frequently used; consult the Library Reference for a complete guide to file objects. @@ -469,7 +469,7 @@ Saving structured data with :mod:`json` .. index:: pair: module; json Strings can easily be written to and read from a file. Numbers take a bit more -effort, since the :meth:`read` method only returns strings, which will have to +effort, since the :meth:`~io.TextIOBase.read` method only returns strings, which will have to be passed to a function like :func:`int`, which takes a string like ``'123'`` and returns its numeric value 123. When you want to save more complex data types like nested lists and dictionaries, parsing and serializing by hand diff --git a/Doc/tutorial/interactive.rst b/Doc/tutorial/interactive.rst index c0eb1feec4eb4d..4e054c4e6c2c32 100644 --- a/Doc/tutorial/interactive.rst +++ b/Doc/tutorial/interactive.rst @@ -23,7 +23,7 @@ Python statement names, the current local variables, and the available module names. For dotted expressions such as ``string.a``, it will evaluate the expression up to the final ``'.'`` and then suggest completions from the attributes of the resulting object. Note that this may execute -application-defined code if an object with a :meth:`__getattr__` method +application-defined code if an object with a :meth:`~object.__getattr__` method is part of the expression. The default configuration also saves your history into a file named :file:`.python_history` in your user directory. The history will be available again during the next interactive interpreter @@ -51,4 +51,4 @@ bpython_. .. _GNU Readline: https://tiswww.case.edu/php/chet/readline/rltop.html .. _IPython: https://ipython.org/ -.. _bpython: https://www.bpython-interpreter.org/ +.. _bpython: https://bpython-interpreter.org/ diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index b71c61089e6dc1..42ebf2b3d294a8 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,13 +10,13 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.12` +The Python interpreter is usually installed as :file:`/usr/local/bin/python3.13` on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.12 + python3.13 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -24,7 +24,7 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) On Windows machines where you have installed Python from the :ref:`Microsoft Store -`, the :file:`python3.12` command will be available. If you have +`, the :file:`python3.13` command will be available. If you have the :ref:`py.exe launcher ` installed, you can use the :file:`py` command. See :ref:`setting-envvars` for other ways to launch Python. @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.12 - Python 3.12 (default, April 4 2022, 09:25:04) + $ python3.13 + Python 3.13 (default, April 4 2023, 09:25:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index ebc2e9187534b4..4536ab9486d39c 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -52,8 +52,8 @@ Numbers The interpreter acts as a simple calculator: you can type an expression at it and it will write the value. Expression syntax is straightforward: the -operators ``+``, ``-``, ``*`` and ``/`` work just like in most other languages -(for example, Pascal or C); parentheses (``()``) can be used for grouping. +operators ``+``, ``-``, ``*`` and ``/`` can be used to perform +arithmetic; parentheses (``()``) can be used for grouping. For example:: >>> 2 + 2 @@ -138,16 +138,25 @@ and uses the ``j`` or ``J`` suffix to indicate the imaginary part .. _tut-strings: -Strings -------- +Text +---- -Besides numbers, Python can also manipulate strings, which can be expressed -in several ways. They can be enclosed in single quotes (``'...'``) or -double quotes (``"..."``) with the same result [#]_. ``\`` can be used -to escape quotes:: +Python can manipulate text (represented by type :class:`str`, so-called +"strings") as well as numbers. This includes characters "``!``", words +"``rabbit``", names "``Paris``", sentences "``Got your back.``", etc. +"``Yay! :)``". They can be enclosed in single quotes (``'...'``) or double +quotes (``"..."``) with the same result [#]_. >>> 'spam eggs' # single quotes 'spam eggs' + >>> "Paris rabbit got your back :)! Yay!" # double quotes + 'Paris rabbit got your back :)! Yay!' + >>> '1975' # digits and numerals enclosed in quotes are also strings + '1975' + +To quote a quote, we need to "escape" it, by preceding it with ``\``. +Alternatively, we can use the other type of quotation marks:: + >>> 'doesn\'t' # use \' to escape the single quote... "doesn't" >>> "doesn't" # ...or use double quotes instead @@ -159,23 +168,14 @@ to escape quotes:: >>> '"Isn\'t," they said.' '"Isn\'t," they said.' -In the interactive interpreter, the output string is enclosed in quotes and -special characters are escaped with backslashes. While this might sometimes -look different from the input (the enclosing quotes could change), the two -strings are equivalent. The string is enclosed in double quotes if -the string contains a single quote and no double quotes, otherwise it is -enclosed in single quotes. The :func:`print` function produces a more -readable output, by omitting the enclosing quotes and by printing escaped -and special characters:: +In the Python shell, the string definition and output string can look +different. The :func:`print` function produces a more readable output, by +omitting the enclosing quotes and by printing escaped and special characters:: - >>> '"Isn\'t," they said.' - '"Isn\'t," they said.' - >>> print('"Isn\'t," they said.') - "Isn't," they said. >>> s = 'First line.\nSecond line.' # \n means newline - >>> s # without print(), \n is included in the output + >>> s # without print(), special characters are included in the string 'First line.\nSecond line.' - >>> print(s) # with print(), \n produces a new line + >>> print(s) # with print(), special characters are interpreted, so \n produces new line First line. Second line. @@ -428,7 +428,7 @@ type, i.e. it is possible to change their content:: [1, 8, 27, 64, 125] You can also add new items at the end of the list, by using -the :meth:`~list.append` *method* (we will see more about methods later):: +the :meth:`!list.append` *method* (we will see more about methods later):: >>> cubes.append(216) # add the cube of 6 >>> cubes.append(7 ** 3) # and the cube of 7 @@ -480,7 +480,7 @@ First Steps Towards Programming Of course, we can use Python for more complicated tasks than adding two and two together. For instance, we can write an initial sub-sequence of the -`Fibonacci series `_ +`Fibonacci series `_ as follows:: >>> # Fibonacci series: diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 3bd034bcc9703f..bf9e8e0b7b8066 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -183,7 +183,7 @@ The Module Search Path .. index:: triple: module; search; path -When a module named :mod:`spam` is imported, the interpreter first searches for +When a module named :mod:`!spam` is imported, the interpreter first searches for a built-in module with that name. These module names are listed in :data:`sys.builtin_module_names`. If not found, it then searches for a file named :file:`spam.py` in a list of directories given by the variable @@ -389,7 +389,7 @@ Packages ======== Packages are a way of structuring Python's module namespace by using "dotted -module names". For example, the module name :mod:`A.B` designates a submodule +module names". For example, the module name :mod:`!A.B` designates a submodule named ``B`` in a package named ``A``. Just like the use of modules saves the authors of different modules from having to worry about each other's global variable names, the use of dotted module names saves the authors of multi-module @@ -448,7 +448,7 @@ example:: import sound.effects.echo -This loads the submodule :mod:`sound.effects.echo`. It must be referenced with +This loads the submodule :mod:`!sound.effects.echo`. It must be referenced with its full name. :: sound.effects.echo.echofilter(input, output, delay=0.7, atten=4) @@ -457,7 +457,7 @@ An alternative way of importing the submodule is:: from sound.effects import echo -This also loads the submodule :mod:`echo`, and makes it available without its +This also loads the submodule :mod:`!echo`, and makes it available without its package prefix, so it can be used as follows:: echo.echofilter(input, output, delay=0.7, atten=4) @@ -466,8 +466,8 @@ Yet another variation is to import the desired function or variable directly:: from sound.effects.echo import echofilter -Again, this loads the submodule :mod:`echo`, but this makes its function -:func:`echofilter` directly available:: +Again, this loads the submodule :mod:`!echo`, but this makes its function +:func:`!echofilter` directly available:: echofilter(input, output, delay=0.7, atten=4) @@ -510,11 +510,27 @@ code:: __all__ = ["echo", "surround", "reverse"] This would mean that ``from sound.effects import *`` would import the three -named submodules of the :mod:`sound.effects` package. +named submodules of the :mod:`!sound.effects` package. + +Be aware that submodules might become shadowed by locally defined names. For +example, if you added a ``reverse`` function to the +:file:`sound/effects/__init__.py` file, the ``from sound.effects import *`` +would only import the two submodules ``echo`` and ``surround``, but *not* the +``reverse`` submodule, because it is shadowed by the locally defined +``reverse`` function:: + + __all__ = [ + "echo", # refers to the 'echo.py' file + "surround", # refers to the 'surround.py' file + "reverse", # !!! refers to the 'reverse' function now !!! + ] + + def reverse(msg: str): # <-- this name shadows the 'reverse.py' submodule + return msg[::-1] # in the case of a 'from sound.effects import *' If ``__all__`` is not defined, the statement ``from sound.effects import *`` -does *not* import all submodules from the package :mod:`sound.effects` into the -current namespace; it only ensures that the package :mod:`sound.effects` has +does *not* import all submodules from the package :mod:`!sound.effects` into the +current namespace; it only ensures that the package :mod:`!sound.effects` has been imported (possibly running any initialization code in :file:`__init__.py`) and then imports whatever names are defined in the package. This includes any names defined (and submodules explicitly loaded) by :file:`__init__.py`. It @@ -525,8 +541,8 @@ previous :keyword:`import` statements. Consider this code:: import sound.effects.surround from sound.effects import * -In this example, the :mod:`echo` and :mod:`surround` modules are imported in the -current namespace because they are defined in the :mod:`sound.effects` package +In this example, the :mod:`!echo` and :mod:`!surround` modules are imported in the +current namespace because they are defined in the :mod:`!sound.effects` package when the ``from...import`` statement is executed. (This also works when ``__all__`` is defined.) @@ -545,15 +561,15 @@ packages. Intra-package References ------------------------ -When packages are structured into subpackages (as with the :mod:`sound` package +When packages are structured into subpackages (as with the :mod:`!sound` package in the example), you can use absolute imports to refer to submodules of siblings -packages. For example, if the module :mod:`sound.filters.vocoder` needs to use -the :mod:`echo` module in the :mod:`sound.effects` package, it can use ``from +packages. For example, if the module :mod:`!sound.filters.vocoder` needs to use +the :mod:`!echo` module in the :mod:`!sound.effects` package, it can use ``from sound.effects import echo``. You can also write relative imports, with the ``from module import name`` form of import statement. These imports use leading dots to indicate the current and -parent packages involved in the relative import. From the :mod:`surround` +parent packages involved in the relative import. From the :mod:`!surround` module for example, you might use:: from . import echo diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 6bae279c5e9cde..63f4b5e1ce0207 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -153,7 +153,7 @@ The :mod:`random` module provides tools for making random selections:: 'apple' >>> random.sample(range(100), 10) # sampling without replacement [30, 83, 16, 4, 8, 81, 41, 50, 18, 33] - >>> random.random() # random float + >>> random.random() # random float from the interval [0.0, 1.0) 0.17970987693706186 >>> random.randrange(6) # random integer chosen from range(6) 4 diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index d1bba098d7d23b..a6dead2eac11f6 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -207,4 +207,6 @@ necessary packages with ``install -r``: ``pip`` has many more options. Consult the :ref:`installing-index` guide for complete documentation for ``pip``. When you've written a package and want to make it available on the Python Package Index, -consult the :ref:`distributing-index` guide. +consult the `Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 9d4042ce5a7e8a..56235bf4c28c7c 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -59,7 +59,7 @@ all consecutive arguments will end up in :data:`sys.argv` -- note that the first element, subscript zero (``sys.argv[0]``), is a string reflecting the program's source. -.. cmdoption:: -c +.. option:: -c Execute the Python code in *command*. *command* can be one or more statements separated by newlines, with significant leading whitespace as in @@ -72,7 +72,7 @@ source. .. audit-event:: cpython.run_command command cmdoption-c -.. cmdoption:: -m +.. option:: -m Search :data:`sys.path` for the named module and execute its contents as the :mod:`__main__` module. @@ -103,13 +103,13 @@ source. :option:`-I` option can be used to run the script in isolated mode where :data:`sys.path` contains neither the current directory nor the user's - site-packages directory. All :envvar:`PYTHON*` environment variables are + site-packages directory. All ``PYTHON*`` environment variables are ignored, too. Many standard library modules contain code that is invoked on their execution as a script. An example is the :mod:`timeit` module:: - python -m timeit -s 'setup here' 'benchmarked code here' + python -m timeit -s "setup here" "benchmarked code here" python -m timeit -h # for details .. audit-event:: cpython.run_module module-name cmdoption-m @@ -161,7 +161,7 @@ source. :option:`-I` option can be used to run the script in isolated mode where :data:`sys.path` contains neither the script's directory nor the user's - site-packages directory. All :envvar:`PYTHON*` environment variables are + site-packages directory. All ``PYTHON*`` environment variables are ignored, too. .. audit-event:: cpython.run_file filename @@ -188,35 +188,35 @@ automatically enabled, if available on your platform (see Generic options ~~~~~~~~~~~~~~~ -.. cmdoption:: -? - -h - --help +.. option:: -? + -h + --help Print a short description of all command line options and corresponding environment variables and exit. -.. cmdoption:: --help-env +.. option:: --help-env Print a short description of Python-specific environment variables and exit. .. versionadded:: 3.11 -.. cmdoption:: --help-xoptions +.. option:: --help-xoptions Print a description of implementation-specific :option:`-X` options and exit. .. versionadded:: 3.11 -.. cmdoption:: --help-all +.. option:: --help-all Print complete usage information and exit. .. versionadded:: 3.11 -.. cmdoption:: -V - --version +.. option:: -V + --version Print the Python version number and exit. Example output could be: @@ -240,7 +240,7 @@ Generic options Miscellaneous options ~~~~~~~~~~~~~~~~~~~~~ -.. cmdoption:: -b +.. option:: -b Issue a warning when comparing :class:`bytes` or :class:`bytearray` with :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the @@ -249,13 +249,13 @@ Miscellaneous options .. versionchanged:: 3.5 Affects comparisons of :class:`bytes` with :class:`int`. -.. cmdoption:: -B +.. option:: -B If given, Python won't try to write ``.pyc`` files on the import of source modules. See also :envvar:`PYTHONDONTWRITEBYTECODE`. -.. cmdoption:: --check-hash-based-pycs default|always|never +.. option:: --check-hash-based-pycs default|always|never Control the validation behavior of hash-based ``.pyc`` files. See :ref:`pyc-invalidation`. When set to ``default``, checked and unchecked @@ -269,7 +269,7 @@ Miscellaneous options option. -.. cmdoption:: -d +.. option:: -d Turn on parser debugging output (for expert only). See also the :envvar:`PYTHONDEBUG` environment variable. @@ -278,15 +278,15 @@ Miscellaneous options it's ignored. -.. cmdoption:: -E +.. option:: -E - Ignore all :envvar:`PYTHON*` environment variables, e.g. + Ignore all ``PYTHON*`` environment variables, e.g. :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. See also the :option:`-P` and :option:`-I` (isolated) options. -.. cmdoption:: -i +.. option:: -i When a script is passed as first argument or the :option:`-c` option is used, enter interactive mode after executing the script or the command, even when @@ -297,20 +297,20 @@ Miscellaneous options raises an exception. See also :envvar:`PYTHONINSPECT`. -.. cmdoption:: -I +.. option:: -I Run Python in isolated mode. This also implies :option:`-E`, :option:`-P` and :option:`-s` options. In isolated mode :data:`sys.path` contains neither the script's directory nor - the user's site-packages directory. All :envvar:`PYTHON*` environment + the user's site-packages directory. All ``PYTHON*`` environment variables are ignored, too. Further restrictions may be imposed to prevent the user from injecting malicious code. .. versionadded:: 3.4 -.. cmdoption:: -O +.. option:: -O Remove assert statements and any code conditional on the value of :const:`__debug__`. Augment the filename for compiled @@ -321,7 +321,7 @@ Miscellaneous options Modify ``.pyc`` filenames according to :pep:`488`. -.. cmdoption:: -OO +.. option:: -OO Do :option:`-O` and also discard docstrings. Augment the filename for compiled (:term:`bytecode`) files by adding ``.opt-2`` before the @@ -331,7 +331,7 @@ Miscellaneous options Modify ``.pyc`` filenames according to :pep:`488`. -.. cmdoption:: -P +.. option:: -P Don't prepend a potentially unsafe path to :data:`sys.path`: @@ -348,21 +348,21 @@ Miscellaneous options .. versionadded:: 3.11 -.. cmdoption:: -q +.. option:: -q Don't display the copyright and version messages even in interactive mode. .. versionadded:: 3.2 -.. cmdoption:: -R +.. option:: -R Turn on hash randomization. This option only has an effect if the :envvar:`PYTHONHASHSEED` environment variable is set to ``0``, since hash randomization is enabled by default. On previous versions of Python, this option turns on hash randomization, - so that the :meth:`__hash__` values of str and bytes objects + so that the :meth:`~object.__hash__` values of str and bytes objects are "salted" with an unpredictable random value. Although they remain constant within an individual Python process, they are not predictable between repeated invocations of Python. @@ -381,7 +381,7 @@ Miscellaneous options .. versionadded:: 3.2.3 -.. cmdoption:: -s +.. option:: -s Don't add the :data:`user site-packages directory ` to :data:`sys.path`. @@ -391,7 +391,7 @@ Miscellaneous options :pep:`370` -- Per user site-packages directory -.. cmdoption:: -S +.. option:: -S Disable the import of the module :mod:`site` and the site-dependent manipulations of :data:`sys.path` that it entails. Also disable these @@ -399,7 +399,7 @@ Miscellaneous options :func:`site.main` if you want them to be triggered). -.. cmdoption:: -u +.. option:: -u Force the stdout and stderr streams to be unbuffered. This option has no effect on the stdin stream. @@ -410,7 +410,7 @@ Miscellaneous options The text layer of the stdout and stderr streams now is unbuffered. -.. cmdoption:: -v +.. option:: -v Print a message each time a module is initialized, showing the place (filename or built-in module) from which it is loaded. When given twice @@ -425,7 +425,7 @@ Miscellaneous options .. _using-on-warnings: -.. cmdoption:: -W arg +.. option:: -W arg Warning control. Python's warning machinery by default prints warning messages to :data:`sys.stderr`. @@ -484,13 +484,13 @@ Miscellaneous options details. -.. cmdoption:: -x +.. option:: -x Skip the first line of the source, allowing use of non-Unix forms of ``#!cmd``. This is intended for a DOS specific hack only. -.. cmdoption:: -X +.. option:: -X Reserved for various implementation-specific options. CPython currently defines the following possible values: @@ -535,17 +535,30 @@ Miscellaneous options location indicators when the interpreter displays tracebacks. See also :envvar:`PYTHONNODEBUGRANGES`. * ``-X frozen_modules`` determines whether or not frozen modules are - ignored by the import machinery. A value of "on" means they get - imported and "off" means they are ignored. The default is "on" + ignored by the import machinery. A value of ``on`` means they get + imported and ``off`` means they are ignored. The default is ``on`` if this is an installed Python (the normal case). If it's under - development (running from the source tree) then the default is "off". - Note that the "importlib_bootstrap" and "importlib_bootstrap_external" - frozen modules are always used, even if this flag is set to "off". + development (running from the source tree) then the default is ``off``. + Note that the :mod:`!importlib_bootstrap` and + :mod:`!importlib_bootstrap_external` frozen modules are always used, even + if this flag is set to ``off``. See also :envvar:`PYTHON_FROZEN_MODULES`. * ``-X perf`` enables support for the Linux ``perf`` profiler. When this option is provided, the ``perf`` profiler will be able to report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`, + :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. + *n* must be greater than or equal to 1. + This option may be useful for users who need to limit CPU resources of a + container system. See also :envvar:`PYTHON_CPU_COUNT`. + If *n* is ``default``, nothing is overridden. + * :samp:`-X presite={package.module}` specifies a module that should be + imported before the :mod:`site` module is executed and before the + :mod:`__main__` module exists. Therefore, the imported module isn't + :mod:`__main__`. This can be used to execute code early during Python + initialization. Python needs to be :ref:`built in debug mode ` + for this option to exist. See also :envvar:`PYTHON_PRESITE`. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -593,11 +606,38 @@ Miscellaneous options .. versionadded:: 3.12 The ``-X perf`` option. + .. versionadded:: 3.13 + The ``-X cpu_count`` option. + + .. versionadded:: 3.13 + The ``-X presite`` option. + +Controlling Color +~~~~~~~~~~~~~~~~~ + +The Python interpreter is configured by default to use colors to highlight +output in certain situations such as when displaying tracebacks. This +behavior can be controlled by setting different environment variables. + +Setting the environment variable ``TERM`` to ``dumb`` will disable color. + +If the environment variable ``FORCE_COLOR`` is set, then color will be +enabled regardless of the value of TERM. This is useful on CI systems which +aren’t terminals but can none-the-less display ANSI escape sequences. + +If the environment variable ``NO_COLOR`` is set, Python will disable all color +in the output. This takes precedence over ``FORCE_COLOR``. + +All these environment variables are used also by other tools to control color +output. To control the color output only in the Python interpreter, the +:envvar:`PYTHON_COLORS` environment variable can be used. This variable takes +precedence over ``NO_COLOR``, which in turn takes precedence over +``FORCE_COLOR``. Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. cmdoption:: -J +.. option:: -J Reserved for use by Jython_. @@ -811,8 +851,8 @@ conflict. Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and :ref:`Distutils installation paths ` for - ``python setup.py install --user``. + and :ref:`installation paths ` for + ``python -m pip install --user``. .. seealso:: @@ -851,9 +891,10 @@ conflict. If this environment variable is set to a non-empty string, :func:`faulthandler.enable` is called at startup: install a handler for - :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and - :const:`SIGILL` signals to dump the Python traceback. This is equivalent to - :option:`-X` ``faulthandler`` option. + :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, + :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` and + :const:`~signal.SIGILL` signals to dump the Python traceback. + This is equivalent to :option:`-X` ``faulthandler`` option. .. versionadded:: 3.3 @@ -897,11 +938,14 @@ conflict. * ``default``: use the :ref:`default memory allocators `. * ``malloc``: use the :c:func:`malloc` function of the C library - for all domains (:c:data:`PYMEM_DOMAIN_RAW`, :c:data:`PYMEM_DOMAIN_MEM`, - :c:data:`PYMEM_DOMAIN_OBJ`). + for all domains (:c:macro:`PYMEM_DOMAIN_RAW`, :c:macro:`PYMEM_DOMAIN_MEM`, + :c:macro:`PYMEM_DOMAIN_OBJ`). * ``pymalloc``: use the :ref:`pymalloc allocator ` for - :c:data:`PYMEM_DOMAIN_MEM` and :c:data:`PYMEM_DOMAIN_OBJ` domains and use - the :c:func:`malloc` function for the :c:data:`PYMEM_DOMAIN_RAW` domain. + :c:macro:`PYMEM_DOMAIN_MEM` and :c:macro:`PYMEM_DOMAIN_OBJ` domains and use + the :c:func:`malloc` function for the :c:macro:`PYMEM_DOMAIN_RAW` domain. + * ``mimalloc``: use the :ref:`mimalloc allocator ` for + :c:macro:`PYMEM_DOMAIN_MEM` and :c:macro:`PYMEM_DOMAIN_OBJ` domains and use + the :c:func:`malloc` function for the :c:macro:`PYMEM_DOMAIN_RAW` domain. Install :ref:`debug hooks `: @@ -909,6 +953,7 @@ conflict. allocators `. * ``malloc_debug``: same as ``malloc`` but also install debug hooks. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. + * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. .. versionchanged:: 3.7 Added the ``"default"`` allocator. @@ -1063,6 +1108,35 @@ conflict. .. versionadded:: 3.12 +.. envvar:: PYTHON_CPU_COUNT + + If this variable is set to a positive integer, it overrides the return + values of :func:`os.cpu_count` and :func:`os.process_cpu_count`. + + See also the :option:`-X cpu_count <-X>` command-line option. + + .. versionadded:: 3.13 + +.. envvar:: PYTHON_FROZEN_MODULES + + If this variable is set to ``on`` or ``off``, it determines whether or not + frozen modules are ignored by the import machinery. A value of ``on`` means + they get imported and ``off`` means they are ignored. The default is ``on`` + for non-debug builds (the normal case) and ``off`` for debug builds. + Note that the :mod:`!importlib_bootstrap` and + :mod:`!importlib_bootstrap_external` frozen modules are always used, even + if this flag is set to ``off``. + + See also the :option:`-X frozen_modules <-X>` command-line option. + + .. versionadded:: 3.13 + +.. envvar:: PYTHON_COLORS + + If this variable is set to ``1``, the interpreter will colorize various kinds + of output. Setting it to ``0`` deactivates this behavior. + + .. versionadded:: 3.13 Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ @@ -1072,13 +1146,33 @@ Debug-mode variables If set, Python will dump objects and reference counts still alive after shutting down the interpreter. - Need Python configured with the :option:`--with-trace-refs` build option. + Needs Python configured with the :option:`--with-trace-refs` build option. -.. envvar:: PYTHONDUMPREFSFILE=FILENAME +.. envvar:: PYTHONDUMPREFSFILE If set, Python will dump objects and reference counts still alive - after shutting down the interpreter into a file called *FILENAME*. + after shutting down the interpreter into a file under the path given + as the value to this environment variable. - Need Python configured with the :option:`--with-trace-refs` build option. + Needs Python configured with the :option:`--with-trace-refs` build option. .. versionadded:: 3.11 + +.. envvar:: PYTHON_PRESITE + + If this variable is set to a module, that module will be imported + early in the interpreter lifecycle, before the :mod:`site` module is + executed, and before the :mod:`__main__` module is created. + Therefore, the imported module is not treated as :mod:`__main__`. + + This can be used to execute code early during Python initialization. + + To import a submodule, use ``package.module`` as the value, like in + an import statement. + + See also the :option:`-X presite <-X>` command-line option, + which takes precedence over this variable. + + Needs Python configured with the :option:`--with-pydebug` build option. + + .. versionadded:: 3.13 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index ce858ab2c8d79e..cb7eda42fe3fad 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -5,29 +5,42 @@ Configure Python Build Requirements ================== -Features required to build CPython: +Features and minimum versions required to build CPython: * A `C11 `_ compiler. `Optional C11 features `_ are not required. +* On Windows, Microsoft Visual Studio 2017 or later is required. + * Support for `IEEE 754 `_ floating point numbers and `floating point Not-a-Number (NaN) `_. * Support for threads. -* OpenSSL 1.1.1 or newer for the :mod:`ssl` and :mod:`hashlib` modules. +* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.9 is the recommended + minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules. -* On Windows, Microsoft Visual Studio 2017 or later is required. +* SQLite 3.15.2 for the :mod:`sqlite3` extension module. + +* Tcl/Tk 8.5.12 for the :mod:`tkinter` module. + +* Autoconf 2.71 and aclocal 1.16.4 are required to regenerate the + :file:`configure` script. + +.. versionchanged:: 3.13: + Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. .. versionchanged:: 3.11 C11 compiler, IEEE 754 and NaN support are now required. On Windows, Visual Studio 2017 or later is required. + Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. .. versionchanged:: 3.10 OpenSSL 1.1.1 is now required. + Require SQLite 3.7.15. .. versionchanged:: 3.7 Thread support and OpenSSL 1.0.2 are now required. @@ -37,18 +50,52 @@ Features required to build CPython: inline`` functions. .. versionchanged:: 3.5 - On Windows, Visual Studio 2015 or later is required. + On Windows, Visual Studio 2015 or later is now required. + Tcl/Tk version 8.4 is now required. + +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". +Generated files +=============== + +To reduce build dependencies, Python source code contains multiple generated +files. Commands to regenerate all generated files:: + + make regen-all + make regen-stdlib-module-names + make regen-limited-abi + make regen-configure + +The ``Makefile.pre.in`` file documents generated files, their inputs, and tools used +to regenerate them. Search for ``regen-*`` make targets. + +configure script +---------------- + +The ``make regen-configure`` command regenerates the ``aclocal.m4`` file and +the ``configure`` script using the ``Tools/build/regen-configure.sh`` shell +script which uses an Ubuntu container to get the same tools versions and have a +reproducible output. + +The container is optional, the following command can be run locally:: + + autoreconf -ivf -Werror + +The generated files can change depending on the exact ``autoconf-archive``, +``aclocal`` and ``pkg-config`` versions. + + .. _configure-options: Configure Options ================= -List all ``./configure`` script options using:: +List all :file:`configure` script options using:: ./configure --help @@ -57,22 +104,22 @@ See also the :file:`Misc/SpecialBuilds.txt` in the Python source distribution. General Options --------------- -.. cmdoption:: --enable-loadable-sqlite-extensions +.. option:: --enable-loadable-sqlite-extensions - Support loadable extensions in the :mod:`_sqlite` extension module (default - is no). + Support loadable extensions in the :mod:`!_sqlite` extension module (default + is no) of the :mod:`sqlite3` module. See the :meth:`sqlite3.Connection.enable_load_extension` method of the :mod:`sqlite3` module. .. versionadded:: 3.6 -.. cmdoption:: --disable-ipv6 +.. option:: --disable-ipv6 Disable IPv6 support (enabled by default if supported), see the :mod:`socket` module. -.. cmdoption:: --enable-big-digits=[15|30] +.. option:: --enable-big-digits=[15|30] Define the size in bits of Python :class:`int` digits: 15 or 30 bits. @@ -82,7 +129,7 @@ General Options See :data:`sys.int_info.bits_per_digit `. -.. cmdoption:: --with-suffix=SUFFIX +.. option:: --with-suffix=SUFFIX Set the Python executable suffix to *SUFFIX*. @@ -95,9 +142,9 @@ General Options The default suffix on WASM platform is one of ``.js``, ``.html`` or ``.wasm``. -.. cmdoption:: --with-tzpath= +.. option:: --with-tzpath= - Select the default time zone search path for :data:`zoneinfo.TZPATH`. + Select the default time zone search path for :const:`zoneinfo.TZPATH`. See the :ref:`Compile-time configuration ` of the :mod:`zoneinfo` module. @@ -107,16 +154,16 @@ General Options .. versionadded:: 3.9 -.. cmdoption:: --without-decimal-contextvar +.. option:: --without-decimal-contextvar Build the ``_decimal`` extension module using a thread-local context rather than a coroutine-local context (default), see the :mod:`decimal` module. - See :data:`decimal.HAVE_CONTEXTVAR` and the :mod:`contextvars` module. + See :const:`decimal.HAVE_CONTEXTVAR` and the :mod:`contextvars` module. .. versionadded:: 3.9 -.. cmdoption:: --with-dbmliborder= +.. option:: --with-dbmliborder= Override order to check db backends for the :mod:`dbm` module @@ -126,7 +173,7 @@ General Options * ``gdbm``; * ``bdb``. -.. cmdoption:: --without-c-locale-coercion +.. option:: --without-c-locale-coercion Disable C locale coercion to a UTF-8 based locale (enabled by default). @@ -134,13 +181,13 @@ General Options See :envvar:`PYTHONCOERCECLOCALE` and the :pep:`538`. -.. cmdoption:: --without-freelists +.. option:: --without-freelists Disable all freelists except the empty tuple singleton. .. versionadded:: 3.11 -.. cmdoption:: --with-platlibdir=DIRNAME +.. option:: --with-platlibdir=DIRNAME Python library directory name (default is ``lib``). @@ -150,7 +197,7 @@ General Options .. versionadded:: 3.9 -.. cmdoption:: --with-wheel-pkg-dir=PATH +.. option:: --with-wheel-pkg-dir=PATH Directory of wheel packages used by the :mod:`ensurepip` module (none by default). @@ -158,11 +205,11 @@ General Options Some Linux distribution packaging policies recommend against bundling dependencies. For example, Fedora installs wheel packages in the ``/usr/share/python-wheels/`` directory and don't install the - :mod:`ensurepip._bundled` package. + :mod:`!ensurepip._bundled` package. .. versionadded:: 3.10 -.. cmdoption:: --with-pkg-config=[check|yes|no] +.. option:: --with-pkg-config=[check|yes|no] Whether configure should use :program:`pkg-config` to detect build dependencies. @@ -173,22 +220,221 @@ General Options .. versionadded:: 3.11 -.. cmdoption:: --enable-pystats +.. option:: --enable-pystats + + Turn on internal Python performance statistics gathering. + + By default, statistics gathering is off. Use ``python3 -X pystats`` command + or set ``PYTHONSTATS=1`` environment variable to turn on statistics + gathering at Python startup. + + At Python exit, dump statistics if statistics gathering was on and not + cleared. + + Effects: + + * Add :option:`-X pystats <-X>` command line option. + * Add :envvar:`!PYTHONSTATS` environment variable. + * Define the ``Py_STATS`` macro. + * Add functions to the :mod:`sys` module: - Turn on internal statistics gathering. + * :func:`!sys._stats_on`: Turns on statistics gathering. + * :func:`!sys._stats_off`: Turns off statistics gathering. + * :func:`!sys._stats_clear`: Clears the statistics. + * :func:`!sys._stats_dump`: Dump statistics to file, and clears the statistics. The statistics will be dumped to a arbitrary (probably unique) file in - ``/tmp/py_stats/``, or ``C:\temp\py_stats\`` on Windows. If that directory - does not exist, results will be printed on stdout. + ``/tmp/py_stats/`` (Unix) or ``C:\temp\py_stats\`` (Windows). If that + directory does not exist, results will be printed on stderr. Use ``Tools/scripts/summarize_stats.py`` to read the stats. + Statistics: + + * Opcode: + + * Specialization: success, failure, hit, deferred, miss, deopt, failures; + * Execution count; + * Pair count. + + * Call: + + * Inlined Python calls; + * PyEval calls; + * Frames pushed; + * Frame object created; + * Eval calls: vector, generator, legacy, function VECTORCALL, build class, + slot, function "ex", API, method. + + * Object: + + * incref and decref; + * interpreter incref and decref; + * allocations: all, 512 bytes, 4 kiB, big; + * free; + * to/from free lists; + * dictionary materialized/dematerialized; + * type cache; + * optimization attemps; + * optimization traces created/executed; + * uops executed. + + * Garbage collector: + + * Garbage collections; + * Objects visited; + * Objects collected. + .. versionadded:: 3.11 +.. _free-threading-build: + +.. option:: --disable-gil + + Enables **experimental** support for running Python without the + :term:`global interpreter lock` (GIL): free threading build. + + Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to + :data:`sys.abiflags`. + + See :pep:`703` "Making the Global Interpreter Lock Optional in CPython". + + .. versionadded:: 3.13 + +.. option:: PKG_CONFIG + + Path to ``pkg-config`` utility. + +.. option:: PKG_CONFIG_LIBDIR +.. option:: PKG_CONFIG_PATH + + ``pkg-config`` options. + + +C compiler options +------------------ + +.. option:: CC + + C compiler command. + +.. option:: CFLAGS + + C compiler flags. + +.. option:: CPP + + C preprocessor command. + +.. option:: CPPFLAGS + + C preprocessor flags, e.g. :samp:`-I{include_dir}`. + + +Linker options +-------------- + +.. option:: LDFLAGS + + Linker flags, e.g. :samp:`-L{library_directory}`. + +.. option:: LIBS + + Libraries to pass to the linker, e.g. :samp:`-l{library}`. + +.. option:: MACHDEP + + Name for machine-dependent library files. + + +Options for third-party dependencies +------------------------------------ + +.. versionadded:: 3.11 + +.. option:: BZIP2_CFLAGS +.. option:: BZIP2_LIBS + + C compiler and linker flags to link Python to ``libbz2``, used by :mod:`bz2` + module, overriding ``pkg-config``. + +.. option:: CURSES_CFLAGS +.. option:: CURSES_LIBS + + C compiler and linker flags for ``libncurses`` or ``libncursesw``, used by + :mod:`curses` module, overriding ``pkg-config``. + +.. option:: GDBM_CFLAGS +.. option:: GDBM_LIBS + + C compiler and linker flags for ``gdbm``. + +.. option:: LIBB2_CFLAGS +.. option:: LIBB2_LIBS + + C compiler and linker flags for ``libb2`` (:ref:`BLAKE2 `), + used by :mod:`hashlib` module, overriding ``pkg-config``. + +.. option:: LIBEDIT_CFLAGS +.. option:: LIBEDIT_LIBS + + C compiler and linker flags for ``libedit``, used by :mod:`readline` module, + overriding ``pkg-config``. + +.. option:: LIBFFI_CFLAGS +.. option:: LIBFFI_LIBS + + C compiler and linker flags for ``libffi``, used by :mod:`ctypes` module, + overriding ``pkg-config``. + +.. option:: LIBLZMA_CFLAGS +.. option:: LIBLZMA_LIBS + + C compiler and linker flags for ``liblzma``, used by :mod:`lzma` module, + overriding ``pkg-config``. + +.. option:: LIBREADLINE_CFLAGS +.. option:: LIBREADLINE_LIBS + + C compiler and linker flags for ``libreadline``, used by :mod:`readline` + module, overriding ``pkg-config``. + +.. option:: LIBSQLITE3_CFLAGS +.. option:: LIBSQLITE3_LIBS + + C compiler and linker flags for ``libsqlite3``, used by :mod:`sqlite3` + module, overriding ``pkg-config``. + +.. option:: LIBUUID_CFLAGS +.. option:: LIBUUID_LIBS + + C compiler and linker flags for ``libuuid``, used by :mod:`uuid` module, + overriding ``pkg-config``. + +.. option:: PANEL_CFLAGS +.. option:: PANEL_LIBS + + C compiler and Linker flags for PANEL, overriding ``pkg-config``. + + C compiler and linker flags for ``libpanel`` or ``libpanelw``, used by + :mod:`curses.panel` module, overriding ``pkg-config``. + +.. option:: TCLTK_CFLAGS +.. option:: TCLTK_LIBS + + C compiler and linker flags for TCLTK, overriding ``pkg-config``. + +.. option:: ZLIB_CFLAGS +.. option:: ZLIB_LIBS + + C compiler and linker flags for ``libzlib``, used by :mod:`gzip` module, + overriding ``pkg-config``. + + WebAssembly Options ------------------- -.. cmdoption:: --with-emscripten-target=[browser|node] +.. option:: --with-emscripten-target=[browser|node] Set build flavor for ``wasm32-emscripten``. @@ -197,7 +443,7 @@ WebAssembly Options .. versionadded:: 3.11 -.. cmdoption:: --enable-wasm-dynamic-linking +.. option:: --enable-wasm-dynamic-linking Turn on dynamic linking support for WASM. @@ -206,7 +452,7 @@ WebAssembly Options .. versionadded:: 3.11 -.. cmdoption:: --enable-wasm-pthreads +.. option:: --enable-wasm-pthreads Turn on pthreads support for WASM. @@ -216,30 +462,30 @@ WebAssembly Options Install Options --------------- -.. cmdoption:: --prefix=PREFIX +.. option:: --prefix=PREFIX Install architecture-independent files in PREFIX. On Unix, it defaults to :file:`/usr/local`. - This value can be retrived at runtime using :data:`sys.prefix`. + This value can be retrieved at runtime using :data:`sys.prefix`. As an example, one can use ``--prefix="$HOME/.local/"`` to install a Python in its home directory. -.. cmdoption:: --exec-prefix=EPREFIX +.. option:: --exec-prefix=EPREFIX Install architecture-dependent files in EPREFIX, defaults to :option:`--prefix`. - This value can be retrived at runtime using :data:`sys.exec_prefix`. + This value can be retrieved at runtime using :data:`sys.exec_prefix`. -.. cmdoption:: --disable-test-modules +.. option:: --disable-test-modules Don't build nor install test modules, like the :mod:`test` package or the - :mod:`_testcapi` extension module (built and installed by default). + :mod:`!_testcapi` extension module (built and installed by default). .. versionadded:: 3.10 -.. cmdoption:: --with-ensurepip=[upgrade|install|no] +.. option:: --with-ensurepip=[upgrade|install|no] Select the :mod:`ensurepip` command run on Python installation: @@ -258,7 +504,7 @@ Configuring Python using ``--enable-optimizations --with-lto`` (PGO + LTO) is recommended for best performance. The experimental ``--enable-bolt`` flag can also be used to improve performance. -.. cmdoption:: --enable-optimizations +.. option:: --enable-optimizations Enable Profile Guided Optimization (PGO) using :envvar:`PROFILE_TASK` (disabled by default). @@ -284,7 +530,10 @@ also be used to improve performance. .. versionadded:: 3.8 -.. cmdoption:: --with-lto=[full|thin|no|yes] + .. versionchanged:: 3.13 + Task failure is no longer ignored silently. + +.. option:: --with-lto=[full|thin|no|yes] Enable Link Time Optimization (LTO) in any build (disabled by default). @@ -299,7 +548,7 @@ also be used to improve performance. .. versionchanged:: 3.12 Use ThinLTO as the default optimization policy on Clang if the compiler accepts the flag. -.. cmdoption:: --enable-bolt +.. option:: --enable-bolt Enable usage of the `BOLT post-link binary optimizer `_ (disabled by @@ -314,22 +563,49 @@ also be used to improve performance. is dependent on a combination of the build environment + the other optimization configure args + the CPU architecture, and not all combinations are supported. + BOLT versions before LLVM 16 are known to crash BOLT under some scenarios. + Use of LLVM 16 or newer for BOLT optimization is strongly encouraged. + + The :envvar:`!BOLT_INSTRUMENT_FLAGS` and :envvar:`!BOLT_APPLY_FLAGS` + :program:`configure` variables can be defined to override the default set of + arguments for :program:`llvm-bolt` to instrument and apply BOLT data to + binaries, respectively. .. versionadded:: 3.12 -.. cmdoption:: --with-computed-gotos +.. option:: BOLT_APPLY_FLAGS + + Arguments to ``llvm-bolt`` when creating a `BOLT optimized binary + `_. + + .. versionadded:: 3.12 + +.. option:: BOLT_INSTRUMENT_FLAGS + + Arguments to ``llvm-bolt`` when instrumenting binaries. + + .. versionadded:: 3.12 + +.. option:: --with-computed-gotos Enable computed gotos in evaluation loop (enabled by default on supported compilers). -.. cmdoption:: --without-pymalloc +.. option:: --without-mimalloc + + Disable the fast mimalloc allocator :ref:`mimalloc ` + (enabled by default). + + See also :envvar:`PYTHONMALLOC` environment variable. + +.. option:: --without-pymalloc Disable the specialized Python memory allocator :ref:`pymalloc ` (enabled by default). See also :envvar:`PYTHONMALLOC` environment variable. -.. cmdoption:: --without-doc-strings +.. option:: --without-doc-strings Disable static documentation strings to reduce the memory footprint (enabled by default). Documentation strings defined in Python are not affected. @@ -338,11 +614,11 @@ also be used to improve performance. See the ``PyDoc_STRVAR()`` macro. -.. cmdoption:: --enable-profiling +.. option:: --enable-profiling Enable C-level code profiling with ``gprof`` (disabled by default). -.. cmdoption:: --with-strict-overflow +.. option:: --with-strict-overflow Add ``-fstrict-overflow`` to the C compiler flags (by default we add ``-fno-strict-overflow`` instead). @@ -361,7 +637,7 @@ Effects of a debug build: * Display all warnings by default: the list of default warning filters is empty in the :mod:`warnings` module. * Add ``d`` to :data:`sys.abiflags`. -* Add :func:`sys.gettotalrefcount` function. +* Add :func:`!sys.gettotalrefcount` function. * Add :option:`-X showrefcount <-X>` command line option. * Add :option:`-d` command line option and :envvar:`PYTHONDEBUG` environment variable to debug the parser. @@ -383,7 +659,7 @@ Effects of a debug build: * Check that deallocator functions don't change the current exception. * The garbage collector (:func:`gc.collect` function) runs some basic checks on objects consistency. - * The :c:macro:`Py_SAFE_DOWNCAST()` macro checks for integer underflow and + * The :c:macro:`!Py_SAFE_DOWNCAST()` macro checks for integer underflow and overflow when downcasting from wide types to narrow types. See also the :ref:`Python Development Mode ` and the @@ -392,34 +668,39 @@ See also the :ref:`Python Development Mode ` and the .. versionchanged:: 3.8 Release builds and debug builds are now ABI compatible: defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the - :option:`--with-trace-refs` option), which introduces the only ABI - incompatibility. + :option:`--with-trace-refs` option). Debug options ------------- -.. cmdoption:: --with-pydebug +.. option:: --with-pydebug :ref:`Build Python in debug mode `: define the ``Py_DEBUG`` macro (disabled by default). -.. cmdoption:: --with-trace-refs +.. option:: --with-trace-refs Enable tracing references for debugging purpose (disabled by default). Effects: * Define the ``Py_TRACE_REFS`` macro. - * Add :func:`sys.getobjects` function. + * Add :func:`!sys.getobjects` function. * Add :envvar:`PYTHONDUMPREFS` environment variable. - This build is not ABI compatible with release build (default build) or debug - build (``Py_DEBUG`` and ``Py_REF_DEBUG`` macros). + The :envvar:`PYTHONDUMPREFS` environment variable can be used to dump + objects and reference counts still alive at Python exit. + + :ref:`Statically allocated objects ` are not traced. + + .. versionchanged:: 3.13 + This build is now ABI compatible with release build and :ref:`debug build + `. .. versionadded:: 3.8 -.. cmdoption:: --with-assertions +.. option:: --with-assertions Build with C assertions enabled (default is no): ``assert(...);`` and ``_PyObject_ASSERT(...);``. @@ -432,11 +713,11 @@ Debug options .. versionadded:: 3.6 -.. cmdoption:: --with-valgrind +.. option:: --with-valgrind Enable Valgrind support (default is no). -.. cmdoption:: --with-dtrace +.. option:: --with-dtrace Enable DTrace support (default is no). @@ -445,19 +726,19 @@ Debug options .. versionadded:: 3.6 -.. cmdoption:: --with-address-sanitizer +.. option:: --with-address-sanitizer Enable AddressSanitizer memory error detector, ``asan`` (default is no). .. versionadded:: 3.6 -.. cmdoption:: --with-memory-sanitizer +.. option:: --with-memory-sanitizer Enable MemorySanitizer allocation error detector, ``msan`` (default is no). .. versionadded:: 3.6 -.. cmdoption:: --with-undefined-behavior-sanitizer +.. option:: --with-undefined-behavior-sanitizer Enable UndefinedBehaviorSanitizer undefined behaviour detector, ``ubsan`` (default is no). @@ -468,11 +749,11 @@ Debug options Linker options -------------- -.. cmdoption:: --enable-shared +.. option:: --enable-shared Enable building a shared Python library: ``libpython`` (default is no). -.. cmdoption:: --without-static-libpython +.. option:: --without-static-libpython Do not build ``libpythonMAJOR.MINOR.a`` and do not install ``python.o`` (built and enabled by default). @@ -483,31 +764,32 @@ Linker options Libraries options ----------------- -.. cmdoption:: --with-libs='lib1 ...' +.. option:: --with-libs='lib1 ...' Link against additional libraries (default is no). -.. cmdoption:: --with-system-expat +.. option:: --with-system-expat - Build the :mod:`pyexpat` module using an installed ``expat`` library + Build the :mod:`!pyexpat` module using an installed ``expat`` library (default is no). -.. cmdoption:: --with-system-libmpdec +.. option:: --with-system-libmpdec Build the ``_decimal`` extension module using an installed ``mpdec`` library, see the :mod:`decimal` module (default is no). .. versionadded:: 3.3 -.. cmdoption:: --with-readline=editline +.. option:: --with-readline=readline|editline - Use ``editline`` library for backend of the :mod:`readline` module. + Designate a backend library for the :mod:`readline` module. - Define the ``WITH_EDITLINE`` macro. + * readline: Use readline as the backend. + * editline: Use editline as the backend. .. versionadded:: 3.10 -.. cmdoption:: --without-readline +.. option:: --without-readline Don't build the :mod:`readline` module (built by default). @@ -515,21 +797,21 @@ Libraries options .. versionadded:: 3.10 -.. cmdoption:: --with-libm=STRING +.. option:: --with-libm=STRING Override ``libm`` math library to *STRING* (default is system-dependent). -.. cmdoption:: --with-libc=STRING +.. option:: --with-libc=STRING Override ``libc`` C library to *STRING* (default is system-dependent). -.. cmdoption:: --with-openssl=DIR +.. option:: --with-openssl=DIR Root of the OpenSSL directory. .. versionadded:: 3.7 -.. cmdoption:: --with-openssl-rpath=[no|auto|DIR] +.. option:: --with-openssl-rpath=[no|auto|DIR] Set runtime library directory (rpath) for OpenSSL libraries: @@ -544,7 +826,7 @@ Libraries options Security Options ---------------- -.. cmdoption:: --with-hash-algorithm=[fnv|siphash13|siphash24] +.. option:: --with-hash-algorithm=[fnv|siphash13|siphash24] Select hash algorithm for use in ``Python/pyhash.c``: @@ -557,7 +839,7 @@ Security Options .. versionadded:: 3.11 ``siphash13`` is added and it is the new default. -.. cmdoption:: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 +.. option:: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 Built-in hash modules: @@ -570,7 +852,7 @@ Security Options .. versionadded:: 3.9 -.. cmdoption:: --with-ssl-default-suites=[python|openssl|STRING] +.. option:: --with-ssl-default-suites=[python|openssl|STRING] Override the OpenSSL default cipher suites string: @@ -592,19 +874,19 @@ macOS Options See ``Mac/README.rst``. -.. cmdoption:: --enable-universalsdk -.. cmdoption:: --enable-universalsdk=SDKDIR +.. option:: --enable-universalsdk +.. option:: --enable-universalsdk=SDKDIR Create a universal binary build. *SDKDIR* specifies which macOS SDK should be used to perform the build (default is no). -.. cmdoption:: --enable-framework -.. cmdoption:: --enable-framework=INSTALLDIR +.. option:: --enable-framework +.. option:: --enable-framework=INSTALLDIR Create a Python.framework rather than a traditional Unix install. Optional *INSTALLDIR* specifies the installation path (default is no). -.. cmdoption:: --with-universal-archs=ARCH +.. option:: --with-universal-archs=ARCH Specify the kind of universal binary that should be created. This option is only valid when :option:`--enable-universalsdk` is set. @@ -620,7 +902,7 @@ See ``Mac/README.rst``. * ``intel-64``; * ``all``. -.. cmdoption:: --with-framework-name=FRAMEWORK +.. option:: --with-framework-name=FRAMEWORK Specify the name for the python framework on macOS only valid when :option:`--enable-framework` is set (default: ``Python``). @@ -634,21 +916,21 @@ for another CPU architecture or platform. Cross compiling requires a Python interpreter for the build platform. The version of the build Python must match the version of the cross compiled host Python. -.. cmdoption:: --build=BUILD +.. option:: --build=BUILD configure for building on BUILD, usually guessed by :program:`config.guess`. -.. cmdoption:: --host=HOST +.. option:: --host=HOST cross-compile to build programs to run on HOST (target platform) -.. cmdoption:: --with-build-python=path/to/python +.. option:: --with-build-python=path/to/python path to build ``python`` binary for cross compiling .. versionadded:: 3.11 -.. cmdoption:: CONFIG_SITE=file +.. option:: CONFIG_SITE=file An environment variable that points to a file with configure overrides. @@ -659,6 +941,12 @@ the version of the cross compiled host Python. ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no +.. option:: HOSTRUNNER + + Program to run CPython for the host platform for cross-compilation. + + .. versionadded:: 3.11 + Cross compiling example:: @@ -679,7 +967,6 @@ Main files of the build system * :file:`pyconfig.h` (created by :file:`configure`); * :file:`Modules/Setup`: C extensions built by the Makefile using :file:`Module/makesetup` shell script; -* :file:`setup.py`: C extensions built using the ``setuptools`` package. Main build steps ---------------- @@ -688,8 +975,7 @@ Main build steps * A static ``libpython`` library (``.a``) is created from objects files. * ``python.o`` and the static ``libpython`` library are linked into the final ``python`` program. -* C extensions are built by the Makefile (see :file:`Modules/Setup`) - and ``python setup.py build``. +* C extensions are built by the Makefile (see :file:`Modules/Setup`). Main Makefile targets --------------------- @@ -701,9 +987,18 @@ Main Makefile targets You can use the configure :option:`--enable-optimizations` option to make this the default target of the ``make`` command (``make all`` or just ``make``). -* ``make buildbottest``: Build Python and run the Python test suite, the same - way than buildbots test Python. Set ``TESTTIMEOUT`` variable (in seconds) - to change the test timeout (1200 by default: 20 minutes). + +* ``make test``: Build Python and run the Python test suite with ``--fast-ci`` + option. Variables: + + * ``TESTOPTS``: additional regrtest command line options. + * ``TESTPYTHONOPTS``: additional Python command line options. + * ``TESTTIMEOUT``: timeout in seconds (default: 20 minutes). + +* ``make buildbottest``: Similar to ``make test``, but use ``--slow-ci`` + option and default timeout of 20 minutes, instead of ``--fast-ci`` option + and a default timeout of 10 minutes. + * ``make install``: Build and install Python. * ``make regen-all``: Regenerate (almost) all generated files; ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately @@ -741,18 +1036,15 @@ Example on Linux x86-64:: At the beginning of the files, C extensions are built as built-in modules. Extensions defined after the ``*shared*`` marker are built as dynamic libraries. -The :file:`setup.py` script only builds C extensions as shared libraries using -the :mod:`distutils` module. - -The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and -:c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined +The :c:macro:`!PyAPI_FUNC()`, :c:macro:`!PyAPI_DATA()` and +:c:macro:`PyMODINIT_FUNC` macros of :file:`Include/exports.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: * Use ``Py_EXPORTED_SYMBOL`` if the ``Py_BUILD_CORE_MODULE`` is defined * Use ``Py_IMPORTED_SYMBOL`` otherwise. If the ``Py_BUILD_CORE_BUILTIN`` macro is used by mistake on a C extension -built as a shared library, its ``PyInit_xxx()`` function is not exported, +built as a shared library, its :samp:`PyInit_{xxx}()` function is not exported, causing an :exc:`ImportError` on import. @@ -773,11 +1065,11 @@ Preprocessor flags .. envvar:: CPPFLAGS - (Objective) C/C++ preprocessor flags, e.g. ``-I`` if you have - headers in a nonstandard directory ````. + (Objective) C/C++ preprocessor flags, e.g. :samp:`-I{include_dir}` if you have + headers in a nonstandard directory *include_dir*. Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's - value for setup.py to be able to build extension modules using the + value to be able to build extension modules using the directories specified in the environment variables. .. envvar:: BASECPPFLAGS @@ -814,8 +1106,8 @@ Compiler flags .. envvar:: CFLAGS_NODIST :envvar:`CFLAGS_NODIST` is used for building the interpreter and stdlib C - extensions. Use it when a compiler flag should *not* be part of the - distutils :envvar:`CFLAGS` once Python is installed (:issue:`21121`). + extensions. Use it when a compiler flag should *not* be part of + :envvar:`CFLAGS` once Python is installed (:gh:`65320`). In particular, :envvar:`CFLAGS` should not contain: @@ -945,7 +1237,7 @@ Linker flags :envvar:`LDFLAGS_NODIST` is used in the same manner as :envvar:`CFLAGS_NODIST`. Use it when a linker flag should *not* be part of - the distutils :envvar:`LDFLAGS` once Python is installed (:issue:`35257`). + :envvar:`LDFLAGS` once Python is installed (:gh:`65320`). In particular, :envvar:`LDFLAGS` should not contain: @@ -963,11 +1255,11 @@ Linker flags .. envvar:: LDFLAGS - Linker flags, e.g. ``-L`` if you have libraries in a nonstandard - directory ````. + Linker flags, e.g. :samp:`-L{lib_dir}` if you have libraries in a nonstandard + directory *lib_dir*. Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's - value for setup.py to be able to build extension modules using the + value to be able to build extension modules using the directories specified in the environment variables. .. envvar:: LIBS diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 69cd5c92d884d0..eb1413af2cbc3d 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -125,13 +125,9 @@ http://www.hashcollision.org/hkn/python/idle_intro/index.html. Installing Additional Python Packages ===================================== -There are several methods to install additional Python packages: +This section has moved to the `Python Packaging User Guide`_. -* Packages can be installed via the standard Python distutils mode (``python - setup.py install``). - -* Many packages can also be installed via the :program:`setuptools` extension - or :program:`pip` wrapper, see https://pip.pypa.io/. +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ GUI Programming on the Mac @@ -144,8 +140,8 @@ the foundation of most modern Mac development. Information on PyObjC is available from https://pypi.org/project/pyobjc/. The standard Python GUI toolkit is :mod:`tkinter`, based on the cross-platform -Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with OS -X by Apple, and the latest version can be downloaded and installed from +Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with +macOS by Apple, and the latest version can be downloaded and installed from https://www.activestate.com; it can also be built from source. *wxPython* is another popular cross-platform GUI toolkit that runs natively on diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 0044eb07f56eec..58838c28e6eb86 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -30,9 +30,9 @@ following links: for Debian users https://en.opensuse.org/Portal:Packaging for OpenSuse users - https://docs-old.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch-creating-rpms.html + https://docs.fedoraproject.org/en-US/package-maintainers/Packaging_Tutorial_GNU_Hello/ for Fedora users - http://www.slackbook.org/html/package-management-making-packages.html + https://slackbook.org/html/package-management-making-packages.html for Slackware users diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 2fc90126482268..1cf438b198a9af 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -35,37 +35,48 @@ your :ref:`Python installation `:: The command, if run with ``-h``, will show the available options:: - usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] - ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - optional arguments: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when symlinks - are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory if it - already exists, before environment creation. - --upgrade Upgrade the environment directory to use this version - of Python, assuming Python has been upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the - latest version in PyPI - - Once an environment has been created, you may wish to activate it, e.g. by - sourcing an activate script in its bin directory. + usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] + [--without-scm-ignore-file] + ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + options: + -h, --help show this help message and exit + --system-site-packages + Give the virtual environment access to the system + site-packages dir. + --symlinks Try to use symlinks rather than copies, when + symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. + --clear Delete the contents of the environment directory if + it already exists, before environment creation. + --upgrade Upgrade the environment directory to use this + version of Python, assuming Python has been upgraded + in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + --prompt PROMPT Provides an alternative prompt prefix for this + environment. + --upgrade-deps Upgrade core dependencies (pip) to the latest + version in PyPI + --without-scm-ignore-file + Skips adding the default SCM ignore file to the + environment directory (the default is a .gitignore + file). + + Once an environment has been created, you may wish to activate it, e.g. by + sourcing an activate script in its bin directory. + +.. versionchanged:: 3.13 + + ``--without-scm-ignore-file`` was added along with creating an ignore file + for ``git`` by default. .. versionchanged:: 3.12 diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 43e3c72f3e1cde..598bf3ca9bcc04 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -867,17 +867,18 @@ For example, if the first line of your script starts with #! /usr/bin/python -The default Python will be located and used. As many Python scripts written -to work on Unix will already have this line, you should find these scripts can -be used by the launcher without modification. If you are writing a new script -on Windows which you hope will be useful on Unix, you should use one of the -shebang lines starting with ``/usr``. +The default Python or an active virtual environment will be located and used. +As many Python scripts written to work on Unix will already have this line, +you should find these scripts can be used by the launcher without modification. +If you are writing a new script on Windows which you hope will be useful on +Unix, you should use one of the shebang lines starting with ``/usr``. Any of the above virtual commands can be suffixed with an explicit version (either just the major version, or the major and minor version). Furthermore the 32-bit version can be requested by adding "-32" after the minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the -32-bit python 3.7. +32-bit Python 3.7. If a virtual environment is active, the version will be +ignored and the environment will be used. .. versionadded:: 3.7 @@ -889,7 +890,14 @@ minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the The "-64" suffix is deprecated, and now implies "any architecture that is not provably i386/32-bit". To request a specific environment, use the new - ``-V:`` argument with the complete tag. + :samp:`-V:{TAG}` argument with the complete tag. + +.. versionchanged:: 3.13 + + Virtual commands referencing ``python`` now prefer an active virtual + environment rather than searching :envvar:`PATH`. This handles cases where + the shebang specifies ``/usr/bin/env python3`` but :file:`python3.exe` is + not present in the active environment. The ``/usr/bin/env`` form of shebang line has one further special property. Before looking for installed Python interpreters, this form will search the @@ -1169,8 +1177,8 @@ following advice will prevent conflicts with other installations: listed. * If you are loading :file:`python3.dll` or :file:`python37.dll` in your own - executable, explicitly call :c:func:`Py_SetPath` or (at least) - :c:func:`Py_SetProgramName` before :c:func:`Py_Initialize`. + executable, explicitly set :c:member:`PyConfig.module_search_paths` before + :c:func:`Py_InitializeFromConfig`. * Clear and/or overwrite :envvar:`PYTHONPATH` and set :envvar:`PYTHONHOME` before launching :file:`python.exe` from your application. @@ -1187,21 +1195,22 @@ Otherwise, your users may experience problems using your application. Note that the first suggestion is the best, as the others may still be susceptible to non-standard paths in the registry and user site-packages. -.. versionchanged:: - 3.6 +.. versionchanged:: 3.6 + + Add ``._pth`` file support and removes ``applocal`` option from + ``pyvenv.cfg``. + +.. versionchanged:: 3.6 - * Adds ``._pth`` file support and removes ``applocal`` option from - ``pyvenv.cfg``. - * Adds ``pythonXX.zip`` as a potential landmark when directly adjacent - to the executable. + Add :file:`python{XX}.zip` as a potential landmark when directly adjacent + to the executable. -.. deprecated:: - 3.6 +.. deprecated:: 3.6 - Modules specified in the registry under ``Modules`` (not ``PythonPath``) - may be imported by :class:`importlib.machinery.WindowsRegistryFinder`. - This finder is enabled on Windows in 3.6.0 and earlier, but may need to - be explicitly added to :attr:`sys.meta_path` in the future. + Modules specified in the registry under ``Modules`` (not ``PythonPath``) + may be imported by :class:`importlib.machinery.WindowsRegistryFinder`. + This finder is enabled on Windows in 3.6.0 and earlier, but may need to + be explicitly added to :data:`sys.meta_path` in the future. Additional modules ================== @@ -1246,8 +1255,8 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. cx_Freeze --------- -`cx_Freeze `_ is a ``distutils`` -extension which wraps Python scripts into executable Windows programs +`cx_Freeze `_ +wraps Python scripts into executable Windows programs (:file:`{*}.exe` files). When you have done this, you can distribute your application without requiring your users to install Python. diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 0eefefd863a68f..6d6e51006d5bd8 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -153,9 +153,9 @@ Lundh. A detailed explanation of the interface was written up as :pep:`100`, significant points about the Unicode interfaces. In Python source code, Unicode strings are written as ``u"string"``. Arbitrary -Unicode characters can be written using a new escape sequence, ``\uHHHH``, where +Unicode characters can be written using a new escape sequence, :samp:`\\u{HHHH}`, where *HHHH* is a 4-digit hexadecimal number from 0000 to FFFF. The existing -``\xHHHH`` escape sequence can also be used, and octal escapes can be used for +:samp:`\\x{HH}` escape sequence can also be used, and octal escapes can be used for characters up to U+01FF, which is represented by ``\777``. Unicode strings, just like regular strings, are an immutable sequence type. @@ -664,16 +664,16 @@ extra set of parentheses to pass both values as a tuple: ``L.append( (1,2) )``. The earlier versions of these methods were more forgiving because they used an old function in Python's C interface to parse their arguments; 2.0 modernizes -them to use :func:`PyArg_ParseTuple`, the current argument parsing function, +them to use :c:func:`PyArg_ParseTuple`, the current argument parsing function, which provides more helpful error messages and treats multi-argument calls as errors. If you absolutely must use 2.0 but can't fix your code, you can edit :file:`Objects/listobject.c` and define the preprocessor symbol ``NO_STRICT_LIST_APPEND`` to preserve the old behaviour; this isn't recommended. Some of the functions in the :mod:`socket` module are still forgiving in this -way. For example, :func:`socket.connect( ('hostname', 25) )` is the correct -form, passing a tuple representing an IP address, but :func:`socket.connect( -'hostname', 25 )` also works. :func:`socket.connect_ex` and :func:`socket.bind` +way. For example, ``socket.connect( ('hostname', 25) )`` is the correct +form, passing a tuple representing an IP address, but ``socket.connect('hostname', 25)`` +also works. :meth:`socket.connect_ex ` and :meth:`socket.bind ` are similarly easy-going. 2.0alpha1 tightened these functions up, but because the documentation actually used the erroneous multiple argument form, many people wrote code which would break with the stricter checking. GvR backed out @@ -766,7 +766,7 @@ file, :file:`Include/pyport.h`. Vladimir Marangozov's long-awaited malloc restructuring was completed, to make it easy to have the Python interpreter use a custom allocator instead of C's -standard :func:`malloc`. For documentation, read the comments in +standard :c:func:`malloc`. For documentation, read the comments in :file:`Include/pymem.h` and :file:`Include/objimpl.h`. For the lengthy discussions during which the interface was hammered out, see the web archives of the 'patches' and 'python-dev' lists at python.org. @@ -794,15 +794,15 @@ are generating Python code would run into this limit. A patch by Charles G. Waldman raises the limit from ``2**16`` to ``2**32``. Three new convenience functions intended for adding constants to a module's -dictionary at module initialization time were added: :func:`PyModule_AddObject`, -:func:`PyModule_AddIntConstant`, and :func:`PyModule_AddStringConstant`. Each +dictionary at module initialization time were added: :c:func:`PyModule_AddObject`, +:c:func:`PyModule_AddIntConstant`, and :c:func:`PyModule_AddStringConstant`. Each of these functions takes a module object, a null-terminated C string containing the name to be added, and a third argument for the value to be assigned to the name. This third argument is, respectively, a Python object, a C long, or a C string. -A wrapper API was added for Unix-style signal handlers. :func:`PyOS_getsig` gets -a signal handler and :func:`PyOS_setsig` will set a new handler. +A wrapper API was added for Unix-style signal handlers. :c:func:`PyOS_getsig` gets +a signal handler and :c:func:`PyOS_setsig` will set a new handler. .. ====================================================================== @@ -1030,9 +1030,9 @@ Module changes Lots of improvements and bugfixes were made to Python's extensive standard library; some of the affected modules include :mod:`readline`, -:mod:`ConfigParser`, :mod:`cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, -:mod:`xmllib`, :mod:`aifc`, :mod:`chunk, wave`, :mod:`random`, :mod:`shelve`, -and :mod:`nntplib`. Consult the CVS logs for the exact patch-by-patch details. +:mod:`ConfigParser`, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, +:mod:`xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, +and :mod:`!nntplib`. Consult the CVS logs for the exact patch-by-patch details. Brian Gallew contributed OpenSSL support for the :mod:`socket` module. OpenSSL is an implementation of the Secure Socket Layer, which encrypts the data being diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index 676da702b39693..f0e1ded75a9d27 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -692,8 +692,8 @@ applied, and 136 bugs fixed; both figures are likely to be underestimates. Some of the more notable changes are: * A specialized object allocator is now optionally available, that should be - faster than the system :func:`malloc` and have less memory overhead. The - allocator uses C's :func:`malloc` function to get large pools of memory, and + faster than the system :c:func:`malloc` and have less memory overhead. The + allocator uses C's :c:func:`!malloc` function to get large pools of memory, and then fulfills smaller memory requests from these pools. It can be enabled by providing the :option:`!--with-pymalloc` option to the :program:`configure` script; see :file:`Objects/obmalloc.c` for the implementation details. @@ -701,13 +701,13 @@ of the more notable changes are: Authors of C extension modules should test their code with the object allocator enabled, because some incorrect code may break, causing core dumps at runtime. There are a bunch of memory allocation functions in Python's C API that have - previously been just aliases for the C library's :func:`malloc` and - :func:`free`, meaning that if you accidentally called mismatched functions, the + previously been just aliases for the C library's :c:func:`malloc` and + :c:func:`free`, meaning that if you accidentally called mismatched functions, the error wouldn't be noticeable. When the object allocator is enabled, these - functions aren't aliases of :func:`malloc` and :func:`free` any more, and + functions aren't aliases of :c:func:`!malloc` and :c:func:`!free` any more, and calling the wrong function to free memory will get you a core dump. For - example, if memory was allocated using :func:`PyMem_New`, it has to be freed - using :func:`PyMem_Del`, not :func:`free`. A few modules included with Python + example, if memory was allocated using :c:macro:`PyMem_New`, it has to be freed + using :c:func:`PyMem_Del`, not :c:func:`!free`. A few modules included with Python fell afoul of this and had to be fixed; doubtless there are more third-party modules that will have the same problem. @@ -717,7 +717,7 @@ of the more notable changes are: complain about its lack of speed, and because it's often been used as a naïve benchmark. The :meth:`readline` method of file objects has therefore been rewritten to be much faster. The exact amount of the speedup will vary from - platform to platform depending on how slow the C library's :func:`getc` was, but + platform to platform depending on how slow the C library's :c:func:`!getc` was, but is around 66%, and potentially much faster on some particular operating systems. Tim Peters did much of the benchmarking and coding for this change, motivated by a discussion in comp.lang.python. @@ -770,7 +770,7 @@ of the more notable changes are: reorganization done by Jeremy Hylton. * C extensions which import other modules have been changed to use - :func:`PyImport_ImportModule`, which means that they will use any import hooks + :c:func:`PyImport_ImportModule`, which means that they will use any import hooks that have been installed. This is also encouraged for third-party extensions that need to import some other module from C code. diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 82aff0be1ed3b3..6dfe79cef00987 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -424,22 +424,22 @@ Another significant addition to 2.2 is an iteration interface at both the C and Python levels. Objects can define how they can be looped over by callers. In Python versions up to 2.1, the usual way to make ``for item in obj`` work is -to define a :meth:`__getitem__` method that looks something like this:: +to define a :meth:`~object.__getitem__` method that looks something like this:: def __getitem__(self, index): return -:meth:`__getitem__` is more properly used to define an indexing operation on an +:meth:`~object.__getitem__` is more properly used to define an indexing operation on an object so that you can write ``obj[5]`` to retrieve the sixth element. It's a bit misleading when you're using this only to support :keyword:`for` loops. Consider some file-like object that wants to be looped over; the *index* parameter is essentially meaningless, as the class probably assumes that a -series of :meth:`__getitem__` calls will be made with *index* incrementing by -one each time. In other words, the presence of the :meth:`__getitem__` method +series of :meth:`~object.__getitem__` calls will be made with *index* incrementing by +one each time. In other words, the presence of the :meth:`~object.__getitem__` method doesn't mean that using ``file[5]`` to randomly access the sixth element will work, though it really should. -In Python 2.2, iteration can be implemented separately, and :meth:`__getitem__` +In Python 2.2, iteration can be implemented separately, and :meth:`~object.__getitem__` methods can be limited to classes that really do support random access. The basic idea of iterators is simple. A new built-in function, ``iter(obj)`` or ``iter(C, sentinel)``, is used to get an iterator. ``iter(obj)`` returns @@ -1078,17 +1078,17 @@ code, none of the changes described here will affect you very much. To upgrade an extension module to the new API, perform the following steps: -* Rename :c:func:`Py_TPFLAGS_GC` to :c:func:`PyTPFLAGS_HAVE_GC`. +* Rename :c:macro:`!Py_TPFLAGS_GC` to :c:macro:`Py_TPFLAGS_HAVE_GC`. * Use :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar` to allocate objects, and :c:func:`PyObject_GC_Del` to deallocate them. -* Rename :c:func:`PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and - :c:func:`PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. +* Rename :c:func:`!PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and + :c:func:`!PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. -* Remove :c:func:`PyGC_HEAD_SIZE` from object size calculations. +* Remove :c:macro:`!PyGC_HEAD_SIZE` from object size calculations. -* Remove calls to :c:func:`PyObject_AS_GC` and :c:func:`PyObject_FROM_GC`. +* Remove calls to :c:func:`!PyObject_AS_GC` and :c:func:`!PyObject_FROM_GC`. * A new ``et`` format sequence was added to :c:func:`PyArg_ParseTuple`; ``et`` takes both a parameter and an encoding name, and converts the parameter to the @@ -1105,11 +1105,11 @@ code, none of the changes described here will affect you very much. expected, and a set of pointers to :c:expr:`PyObject*` variables that will be filled in with argument values. -* Two new flags :const:`METH_NOARGS` and :const:`METH_O` are available in method +* Two new flags :c:macro:`METH_NOARGS` and :c:macro:`METH_O` are available in method definition tables to simplify implementation of methods with no arguments or a single untyped argument. Calling such methods is more efficient than calling a - corresponding method that uses :const:`METH_VARARGS`. Also, the old - :const:`METH_OLDARGS` style of writing C methods is now officially deprecated. + corresponding method that uses :c:macro:`METH_VARARGS`. Also, the old + :c:macro:`!METH_OLDARGS` style of writing C methods is now officially deprecated. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new @@ -1219,7 +1219,7 @@ Some of the more notable changes are: operator, but these features were rarely used and therefore buggy. The :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` attributes are also being deprecated. At the C level, the fourth argument to - the :c:func:`PyRange_New` function, ``repeat``, has also been deprecated. + the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. * There were a bunch of patches to the dictionary implementation, mostly to fix potential core dumps if a dictionary contains objects that sneakily changed diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 4eb864f5092d30..c989e6d3fa5787 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -925,7 +925,7 @@ Deletion is more straightforward:: >>> a [1, 3] -One can also now pass slice objects to the :meth:`__getitem__` methods of the +One can also now pass slice objects to the :meth:`~object.__getitem__` methods of the built-in sequences:: >>> range(10).__getitem__(slice(0, 5, 2)) @@ -1392,7 +1392,7 @@ complete list of changes, or look through the CVS logs for all the details. details. * The old and never-documented :mod:`linuxaudiodev` module has been deprecated, - and a new version named :mod:`ossaudiodev` has been added. The module was + and a new version named :mod:`!ossaudiodev` has been added. The module was renamed because the OSS sound drivers can be used on platforms other than Linux, and the interface has also been tidied and brought up to date in various ways. (Contributed by Greg Ward and Nicholas FitzRoy-Dale.) @@ -1474,7 +1474,7 @@ complete list of changes, or look through the CVS logs for all the details. * On Windows, the :mod:`socket` module now ships with Secure Sockets Layer (SSL) support. -* The value of the C :const:`PYTHON_API_VERSION` macro is now exposed at the +* The value of the C :c:macro:`PYTHON_API_VERSION` macro is now exposed at the Python level as ``sys.api_version``. The current exception can be cleared by calling the new :func:`sys.exc_clear` function. @@ -1555,7 +1555,7 @@ complete list of changes, or look through the CVS logs for all the details. # [0.36831796169281006, 0.37441694736480713, 0.35304892063140869] # [0.17574405670166016, 0.18193507194519043, 0.17565798759460449] -* The :mod:`Tix` module has received various bug fixes and updates for the +* The :mod:`!Tix` module has received various bug fixes and updates for the current version of the Tix package. * The :mod:`Tkinter` module now works with a thread-enabled version of Tcl. @@ -1596,7 +1596,7 @@ complete list of changes, or look through the CVS logs for all the details. module. Adding the mix-in as a superclass provides the full dictionary interface - whenever the class defines :meth:`__getitem__`, :meth:`__setitem__`, + whenever the class defines :meth:`~object.__getitem__`, :meth:`__setitem__`, :meth:`__delitem__`, and :meth:`keys`. For example:: >>> import UserDict @@ -1847,7 +1847,7 @@ specifically for allocating Python objects. :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`, and :c:func:`PyObject_Free`. * To allocate and free Python objects, use the "object" family - :c:func:`PyObject_New`, :c:func:`PyObject_NewVar`, and :c:func:`PyObject_Del`. + :c:macro:`PyObject_New`, :c:macro:`PyObject_NewVar`, and :c:func:`PyObject_Del`. Thanks to lots of work by Tim Peters, pymalloc in 2.3 also provides debugging features to catch memory overwrites and doubled frees in both extension modules @@ -1886,10 +1886,10 @@ Changes to Python's build process and to the C API include: (:file:`libpython2.3.so`) by supplying :option:`!--enable-shared` when running Python's :program:`configure` script. (Contributed by Ondrej Palkovsky.) -* The :c:macro:`DL_EXPORT` and :c:macro:`DL_IMPORT` macros are now deprecated. +* The :c:macro:`!DL_EXPORT` and :c:macro:`!DL_IMPORT` macros are now deprecated. Initialization functions for Python extension modules should now be declared using the new macro :c:macro:`PyMODINIT_FUNC`, while the Python core will - generally use the :c:macro:`PyAPI_FUNC` and :c:macro:`PyAPI_DATA` macros. + generally use the :c:macro:`!PyAPI_FUNC` and :c:macro:`!PyAPI_DATA` macros. * The interpreter can be compiled without any docstrings for the built-in functions and modules by supplying :option:`!--without-doc-strings` to the @@ -1897,12 +1897,12 @@ Changes to Python's build process and to the C API include: but will also mean that you can't get help for Python's built-ins. (Contributed by Gustavo Niemeyer.) -* The :c:func:`PyArg_NoArgs` macro is now deprecated, and code that uses it +* The :c:func:`!PyArg_NoArgs` macro is now deprecated, and code that uses it should be changed. For Python 2.2 and later, the method definition table can - specify the :const:`METH_NOARGS` flag, signalling that there are no arguments, + specify the :c:macro:`METH_NOARGS` flag, signalling that there are no arguments, and the argument checking can then be removed. If compatibility with pre-2.2 versions of Python is important, the code could use ``PyArg_ParseTuple(args, - "")`` instead, but this will be slower than using :const:`METH_NOARGS`. + "")`` instead, but this will be slower than using :c:macro:`METH_NOARGS`. * :c:func:`PyArg_ParseTuple` accepts new format characters for various sizes of unsigned integers: ``B`` for :c:expr:`unsigned char`, ``H`` for :c:expr:`unsigned @@ -1918,7 +1918,7 @@ Changes to Python's build process and to the C API include: seconds, according to one measurement). * It's now possible to define class and static methods for a C extension type by - setting either the :const:`METH_CLASS` or :const:`METH_STATIC` flags in a + setting either the :c:macro:`METH_CLASS` or :c:macro:`METH_STATIC` flags in a method's :c:type:`PyMethodDef` structure. * Python now includes a copy of the Expat XML parser's source code, removing any @@ -1998,13 +1998,13 @@ Some of the more notable changes are: It would be difficult to detect any resulting difference from Python code, apart from a slight speed up when Python is run without :option:`-O`. - C extensions that access the :attr:`f_lineno` field of frame objects should + C extensions that access the :attr:`~frame.f_lineno` field of frame objects should instead call ``PyCode_Addr2Line(f->f_code, f->f_lasti)``. This will have the added effect of making the code work as desired under "python -O" in earlier versions of Python. A nifty new feature is that trace functions can now assign to the - :attr:`f_lineno` attribute of frame objects, changing the line that will be + :attr:`~frame.f_lineno` attribute of frame objects, changing the line that will be executed next. A ``jump`` command has been added to the :mod:`pdb` debugger taking advantage of this new feature. (Implemented by Richie Hindle.) diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 98dc83fe935d5e..cab321c3e54d18 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1191,7 +1191,7 @@ complete list of changes, or look through the CVS logs for all the details. effect is to make :file:`.pyc` files significantly smaller. (Contributed by Martin von Löwis.) -* The :mod:`nntplib` module's :class:`NNTP` class gained :meth:`description` and +* The :mod:`!nntplib` module's :class:`NNTP` class gained :meth:`description` and :meth:`descriptions` methods to retrieve newsgroup descriptions for a single group or for a range of groups. (Contributed by Jürgen A. Erhard.) @@ -1468,7 +1468,7 @@ Some of the changes to Python's build process and to the C API are: *X* is a NaN. (Contributed by Tim Peters.) * C code can avoid unnecessary locking by using the new - :c:func:`PyEval_ThreadsInitialized` function to tell if any thread operations + :c:func:`!PyEval_ThreadsInitialized` function to tell if any thread operations have been performed. If this function returns false, no lock operations are needed. (Contributed by Nick Coghlan.) @@ -1476,7 +1476,7 @@ Some of the changes to Python's build process and to the C API are: :c:func:`PyArg_ParseTupleAndKeywords` but takes a :c:type:`va_list` instead of a number of arguments. (Contributed by Greg Chapman.) -* A new method flag, :const:`METH_COEXISTS`, allows a function defined in slots +* A new method flag, :c:macro:`METH_COEXIST`, allows a function defined in slots to co-exist with a :c:type:`PyCFunction` having the same name. This can halve the access time for a method such as :meth:`set.__contains__`. (Contributed by Raymond Hettinger.) @@ -1491,7 +1491,7 @@ Some of the changes to Python's build process and to the C API are: though that processor architecture doesn't call that register "the TSC register". (Contributed by Jeremy Hylton.) -* The :c:type:`tracebackobject` type has been renamed to +* The :c:type:`!tracebackobject` type has been renamed to :c:type:`PyTracebackObject`. .. ====================================================================== diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index dcfaef6ed29494..64b951da3fd5b8 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -575,15 +575,15 @@ structure is:: with-block The expression is evaluated, and it should result in an object that supports the -context management protocol (that is, has :meth:`__enter__` and :meth:`__exit__` +context management protocol (that is, has :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -The object's :meth:`__enter__` is called before *with-block* is executed and +The object's :meth:`~object.__enter__` is called before *with-block* is executed and therefore can run set-up code. It also may return a value that is bound to the name *variable*, if given. (Note carefully that *variable* is *not* assigned the result of *expression*.) -After execution of the *with-block* is finished, the object's :meth:`__exit__` +After execution of the *with-block* is finished, the object's :meth:`~object.__exit__` method is called, even if the block raised an exception, and can therefore run clean-up code. @@ -609,7 +609,7 @@ part-way through the block. .. note:: In this case, *f* is the same object created by :func:`open`, because - :meth:`file.__enter__` returns *self*. + :meth:`~object.__enter__` returns *self*. The :mod:`threading` module's locks and condition variables also support the ':keyword:`with`' statement:: @@ -652,10 +652,10 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`__enter__` and :meth:`__exit__` + manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -* The context manager's :meth:`__enter__` method is called. The value returned +* The context manager's :meth:`~object.__enter__` method is called. The value returned is assigned to *VAR*. If no ``'as VAR'`` clause is present, the value is simply discarded. @@ -669,7 +669,7 @@ A high-level explanation of the context management protocol is: if you do the author of the code containing the ':keyword:`with`' statement will never realize anything went wrong. -* If *BLOCK* didn't raise an exception, the :meth:`__exit__` method is still +* If *BLOCK* didn't raise an exception, the :meth:`~object.__exit__` method is still called, but *type*, *value*, and *traceback* are all ``None``. Let's think through an example. I won't present detailed code but will only @@ -703,7 +703,7 @@ rolled back if there's an exception. Here's the basic interface for def rollback (self): "Rolls back current transaction" -The :meth:`__enter__` method is pretty easy, having only to start a new +The :meth:`~object.__enter__` method is pretty easy, having only to start a new transaction. For this application the resulting cursor object would be a useful result, so the method will return it. The user can then add ``as cursor`` to their ':keyword:`with`' statement to bind the cursor to a variable name. :: @@ -715,7 +715,7 @@ their ':keyword:`with`' statement to bind the cursor to a variable name. :: cursor = self.cursor() return cursor -The :meth:`__exit__` method is the most complicated because it's where most of +The :meth:`~object.__exit__` method is the most complicated because it's where most of the work has to be done. The method has to check if an exception occurred. If there was no exception, the transaction is committed. The transaction is rolled back if there was an exception. @@ -748,10 +748,10 @@ are useful for writing objects for use with the ':keyword:`with`' statement. The decorator is called :func:`contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`__enter__` method, and the value yielded will be the method's return +:meth:`~object.__enter__` method, and the value yielded will be the method's return value that will get bound to the variable in the ':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after the :keyword:`yield` will be -executed in the :meth:`__exit__` method. Any exception raised in the block will +executed in the :meth:`~object.__exit__` method. Any exception raised in the block will be raised by the :keyword:`!yield` statement. Our database example from the previous section could be written using this @@ -954,7 +954,7 @@ The return value must be either a Python integer or long integer. The interpreter will check that the type returned is correct, and raises a :exc:`TypeError` if this requirement isn't met. -A corresponding :attr:`nb_index` slot was added to the C-level +A corresponding :c:member:`~PyNumberMethods.nb_index` slot was added to the C-level :c:type:`PyNumberMethods` structure to let C extensions implement this protocol. ``PyNumber_Index(obj)`` can be used in extension code to call the :meth:`__index__` function and retrieve its result. @@ -1222,7 +1222,7 @@ Here's a partial list of the most notable changes, sorted alphabetically by module name. Consult the :file:`Misc/NEWS` file in the source tree for a more complete list of changes, or look through the SVN logs for all the details. -* The :mod:`audioop` module now supports the a-LAW encoding, and the code for +* The :mod:`!audioop` module now supports the a-LAW encoding, and the code for u-LAW encoding has been improved. (Contributed by Lars Immisch.) * The :mod:`codecs` module gained support for incremental codecs. The @@ -1347,7 +1347,7 @@ complete list of changes, or look through the SVN logs for all the details. :func:`input` function to allow opening files in binary or :term:`universal newlines` mode. Another new parameter, *openhook*, lets you use a function other than :func:`open` to open the input files. Once you're iterating over - the set of files, the :class:`FileInput` object's new :meth:`fileno` returns + the set of files, the :class:`FileInput` object's new :meth:`~fileinput.fileno` returns the file descriptor for the currently opened file. (Contributed by Georg Brandl.) @@ -1422,13 +1422,13 @@ complete list of changes, or look through the SVN logs for all the details. (Contributed by Gregory K. Johnson. Funding was provided by Google's 2005 Summer of Code.) -* New module: the :mod:`msilib` module allows creating Microsoft Installer +* New module: the :mod:`!msilib` module allows creating Microsoft Installer :file:`.msi` files and CAB files. Some support for reading the :file:`.msi` database is also included. (Contributed by Martin von Löwis.) -* The :mod:`nis` module now supports accessing domains other than the system - default domain by supplying a *domain* argument to the :func:`nis.match` and - :func:`nis.maps` functions. (Contributed by Ben Bell.) +* The :mod:`!nis` module now supports accessing domains other than the system + default domain by supplying a *domain* argument to the :func:`!nis.match` and + :func:`!nis.maps` functions. (Contributed by Ben Bell.) * The :mod:`operator` module's :func:`itemgetter` and :func:`attrgetter` functions now support multiple fields. A call such as @@ -1448,10 +1448,10 @@ complete list of changes, or look through the SVN logs for all the details. return times that are precise to fractions of a second; not all systems support such precision.) - Constants named :attr:`os.SEEK_SET`, :attr:`os.SEEK_CUR`, and - :attr:`os.SEEK_END` have been added; these are the parameters to the + Constants named :const:`os.SEEK_SET`, :const:`os.SEEK_CUR`, and + :const:`os.SEEK_END` have been added; these are the parameters to the :func:`os.lseek` function. Two new constants for locking are - :attr:`os.O_SHLOCK` and :attr:`os.O_EXLOCK`. + :const:`os.O_SHLOCK` and :const:`os.O_EXLOCK`. Two new functions, :func:`wait3` and :func:`wait4`, were added. They're similar the :func:`waitpid` function which waits for a child process to exit and returns @@ -1542,7 +1542,7 @@ complete list of changes, or look through the SVN logs for all the details. :meth:`getproto` accessor methods to retrieve the family, type, and protocol values for the socket. -* New module: the :mod:`spwd` module provides functions for accessing the shadow +* New module: the :mod:`!spwd` module provides functions for accessing the shadow password database on systems that support shadow passwords. * The :mod:`struct` is now faster because it compiles format strings into @@ -1602,7 +1602,7 @@ complete list of changes, or look through the SVN logs for all the details. * The :mod:`unicodedata` module has been updated to use version 4.1.0 of the Unicode character database. Version 3.2.0 is required by some specifications, - so it's still available as :attr:`unicodedata.ucd_3_2_0`. + so it's still available as :data:`unicodedata.ucd_3_2_0`. * New module: the :mod:`uuid` module generates universally unique identifiers (UUIDs) according to :rfc:`4122`. The RFC defines several different UUID @@ -2119,9 +2119,9 @@ Changes to Python's build process and to the C API include: the various AST nodes in :file:`Parser/Python.asdl`. A Python script reads this file and generates a set of C structure definitions in :file:`Include/Python-ast.h`. The :c:func:`PyParser_ASTFromString` and - :c:func:`PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take + :c:func:`!PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take Python source as input and return the root of an AST representing the contents. - This AST can then be turned into a code object by :c:func:`PyAST_Compile`. For + This AST can then be turned into a code object by :c:func:`!PyAST_Compile`. For more information, read the source code, and then ask questions on python-dev. The AST code was developed under Jeremy Hylton's management, and implemented by @@ -2151,8 +2151,8 @@ Changes to Python's build process and to the C API include: Previously these different families all reduced to the platform's :c:func:`malloc` and :c:func:`free` functions. This meant it didn't matter if - you got things wrong and allocated memory with the :c:func:`PyMem` function but - freed it with the :c:func:`PyObject` function. With 2.5's changes to obmalloc, + you got things wrong and allocated memory with the ``PyMem`` function but + freed it with the ``PyObject`` function. With 2.5's changes to obmalloc, these families now do different things and mismatches will probably result in a segfault. You should carefully test your C extension modules with Python 2.5. @@ -2172,7 +2172,7 @@ Changes to Python's build process and to the C API include: ``Py_LOCAL(type)`` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the - function be inlined. If :c:func:`PY_LOCAL_AGGRESSIVE` is defined before + function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these optimizations actually make the code faster. (Contributed by Fredrik Lundh at @@ -2181,7 +2181,7 @@ Changes to Python's build process and to the C API include: * ``PyErr_NewException(name, base, dict)`` can now accept a tuple of base classes as its *base* argument. (Contributed by Georg Brandl.) -* The :c:func:`PyErr_Warn` function for issuing warnings is now deprecated in +* The :c:func:`!PyErr_Warn` function for issuing warnings is now deprecated in favour of ``PyErr_WarnEx(category, message, stacklevel)`` which lets you specify the number of stack frames separating this function and the caller. A *stacklevel* of 1 is the function calling :c:func:`PyErr_WarnEx`, 2 is the @@ -2191,7 +2191,7 @@ Changes to Python's build process and to the C API include: compiled with a C++ compiler without errors. (Implemented by Anthony Baxter, Martin von Löwis, Skip Montanaro.) -* The :c:func:`PyRange_New` function was removed. It was never documented, never +* The :c:func:`!PyRange_New` function was removed. It was never documented, never used in the core code, and had dangerously lax error checking. In the unlikely case that your extensions were using it, you can replace it by something like the following:: diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 84bb651e68eed5..e8c1709c42abac 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -121,11 +121,11 @@ about features that will be removed in Python 3.0. You can run code with this switch to see how much work will be necessary to port code to 3.0. The value of this switch is available to Python code as the boolean variable :data:`sys.py3kwarning`, -and to C extension code as :c:data:`Py_Py3kWarningFlag`. +and to C extension code as :c:data:`!Py_Py3kWarningFlag`. .. seealso:: - The 3xxx series of PEPs, which contains proposals for Python 3.0. + The 3\ *xxx* series of PEPs, which contains proposals for Python 3.0. :pep:`3000` describes the development process for Python 3.0. Start with :pep:`3100` that describes the general goals for Python 3.0, and then explore the higher-numbered PEPS that propose @@ -269,15 +269,15 @@ structure is:: with-block The expression is evaluated, and it should result in an object that supports the -context management protocol (that is, has :meth:`__enter__` and :meth:`__exit__` +context management protocol (that is, has :meth:`~object.__enter__` and :meth:`~object.__exit__` methods). -The object's :meth:`__enter__` is called before *with-block* is executed and +The object's :meth:`~object.__enter__` is called before *with-block* is executed and therefore can run set-up code. It also may return a value that is bound to the name *variable*, if given. (Note carefully that *variable* is *not* assigned the result of *expression*.) -After execution of the *with-block* is finished, the object's :meth:`__exit__` +After execution of the *with-block* is finished, the object's :meth:`~object.__exit__` method is called, even if the block raised an exception, and can therefore run clean-up code. @@ -296,7 +296,7 @@ part-way through the block. .. note:: In this case, *f* is the same object created by :func:`open`, because - :meth:`file.__enter__` returns *self*. + :meth:`~object.__enter__` returns *self*. The :mod:`threading` module's locks and condition variables also support the ':keyword:`with`' statement:: @@ -339,16 +339,16 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`__enter__` and :meth:`__exit__` + manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -* The context manager's :meth:`__enter__` method is called. The value returned +* The context manager's :meth:`~object.__enter__` method is called. The value returned is assigned to *VAR*. If no ``as VAR`` clause is present, the value is simply discarded. * The code in *BLOCK* is executed. -* If *BLOCK* raises an exception, the context manager's :meth:`__exit__` method +* If *BLOCK* raises an exception, the context manager's :meth:`~object.__exit__` method is called with three arguments, the exception details (``type, value, traceback``, the same values returned by :func:`sys.exc_info`, which can also be ``None`` if no exception occurred). The method's return value controls whether an exception @@ -357,7 +357,7 @@ A high-level explanation of the context management protocol is: if you do the author of the code containing the ':keyword:`with`' statement will never realize anything went wrong. -* If *BLOCK* didn't raise an exception, the :meth:`__exit__` method is still +* If *BLOCK* didn't raise an exception, the :meth:`~object.__exit__` method is still called, but *type*, *value*, and *traceback* are all ``None``. Let's think through an example. I won't present detailed code but will only @@ -391,7 +391,7 @@ rolled back if there's an exception. Here's the basic interface for def rollback(self): "Rolls back current transaction" -The :meth:`__enter__` method is pretty easy, having only to start a new +The :meth:`~object.__enter__` method is pretty easy, having only to start a new transaction. For this application the resulting cursor object would be a useful result, so the method will return it. The user can then add ``as cursor`` to their ':keyword:`with`' statement to bind the cursor to a variable name. :: @@ -403,7 +403,7 @@ their ':keyword:`with`' statement to bind the cursor to a variable name. :: cursor = self.cursor() return cursor -The :meth:`__exit__` method is the most complicated because it's where most of +The :meth:`~object.__exit__` method is the most complicated because it's where most of the work has to be done. The method has to check if an exception occurred. If there was no exception, the transaction is committed. The transaction is rolled back if there was an exception. @@ -436,10 +436,10 @@ are useful when writing objects for use with the ':keyword:`with`' statement. The decorator is called :func:`contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`__enter__` method, and the value yielded will be the method's return +:meth:`~object.__enter__` method, and the value yielded will be the method's return value that will get bound to the variable in the ':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after the :keyword:`!yield` will be -executed in the :meth:`__exit__` method. Any exception raised in the block will +executed in the :meth:`~object.__exit__` method. Any exception raised in the block will be raised by the :keyword:`!yield` statement. Using this decorator, our database example from the previous section @@ -875,11 +875,11 @@ The signature of the new function is:: The parameters are: - * *args*: positional arguments whose values will be printed out. - * *sep*: the separator, which will be printed between arguments. - * *end*: the ending text, which will be printed after all of the - arguments have been output. - * *file*: the file object to which the output will be sent. +* *args*: positional arguments whose values will be printed out. +* *sep*: the separator, which will be printed between arguments. +* *end*: the ending text, which will be printed after all of the + arguments have been output. +* *file*: the file object to which the output will be sent. .. seealso:: @@ -977,7 +977,7 @@ can be used to include Unicode characters:: print len(s) # 12 Unicode characters At the C level, Python 3.0 will rename the existing 8-bit -string type, called :c:type:`PyStringObject` in Python 2.x, +string type, called :c:type:`!PyStringObject` in Python 2.x, to :c:type:`PyBytesObject`. Python 2.6 uses ``#define`` to support using the names :c:func:`PyBytesObject`, :c:func:`PyBytes_Check`, :c:func:`PyBytes_FromStringAndSize`, @@ -1138,13 +1138,13 @@ indicate that the external caller is done. The *flags* argument to :c:func:`PyObject_GetBuffer` specifies constraints upon the memory returned. Some examples are: - * :const:`PyBUF_WRITABLE` indicates that the memory must be writable. +* :c:macro:`PyBUF_WRITABLE` indicates that the memory must be writable. - * :const:`PyBUF_LOCK` requests a read-only or exclusive lock on the memory. +* :c:macro:`PyBUF_LOCK` requests a read-only or exclusive lock on the memory. - * :const:`PyBUF_C_CONTIGUOUS` and :const:`PyBUF_F_CONTIGUOUS` - requests a C-contiguous (last dimension varies the fastest) or - Fortran-contiguous (first dimension varies the fastest) array layout. +* :c:macro:`PyBUF_C_CONTIGUOUS` and :c:macro:`PyBUF_F_CONTIGUOUS` + requests a C-contiguous (last dimension varies the fastest) or + Fortran-contiguous (first dimension varies the fastest) array layout. Two new argument codes for :c:func:`PyArg_ParseTuple`, ``s*`` and ``z*``, return locked buffer objects for a parameter. @@ -1677,8 +1677,9 @@ Some smaller changes made to the core Python language are: (:issue:`1591665`) * Instance method objects have new attributes for the object and function - comprising the method; the new synonym for :attr:`im_self` is - :attr:`__self__`, and :attr:`im_func` is also available as :attr:`__func__`. + comprising the method; the new synonym for :attr:`!im_self` is + :attr:`~method.__self__`, and :attr:`!im_func` is also available as + :attr:`~method.__func__`. The old names are still supported in Python 2.6, but are gone in 3.0. * An obscure change: when you use the :func:`locals` function inside a @@ -1737,7 +1738,7 @@ Optimizations (Contributed by Antoine Pitrou.) Memory usage is reduced by using pymalloc for the Unicode string's data. -* The ``with`` statement now stores the :meth:`__exit__` method on the stack, +* The ``with`` statement now stores the :meth:`~object.__exit__` method on the stack, producing a small speedup. (Implemented by Jeffrey Yasskin.) * To reduce memory usage, the garbage collector will now clear internal @@ -1805,15 +1806,15 @@ changes, or look through the Subversion logs for all the details. available, instead of restricting itself to protocol 1. (Contributed by W. Barnes.) -* The :mod:`cgi` module will now read variables from the query string +* The :mod:`!cgi` module will now read variables from the query string of an HTTP POST request. This makes it possible to use form actions with URLs that include query strings such as "/cgi-bin/add.py?category=1". (Contributed by Alexandre Fiori and Nubis; :issue:`1817`.) The :func:`parse_qs` and :func:`parse_qsl` functions have been - relocated from the :mod:`cgi` module to the :mod:`urlparse` module. - The versions still available in the :mod:`cgi` module will + relocated from the :mod:`!cgi` module to the :mod:`urlparse` module. + The versions still available in the :mod:`!cgi` module will trigger :exc:`PendingDeprecationWarning` messages in 2.6 (:issue:`600362`). @@ -1850,8 +1851,8 @@ changes, or look through the Subversion logs for all the details. special values and floating-point exceptions in a manner consistent with Annex 'G' of the C99 standard. -* A new data type in the :mod:`collections` module: :class:`namedtuple(typename, - fieldnames)` is a factory function that creates subclasses of the standard tuple +* A new data type in the :mod:`collections` module: ``namedtuple(typename, fieldnames)`` + is a factory function that creates subclasses of the standard tuple whose fields are accessible by name as well as index. For example:: >>> var_type = collections.namedtuple('variable', @@ -1873,7 +1874,7 @@ changes, or look through the Subversion logs for all the details. variable(id=1, name='amplitude', type='int', size=4) Several places in the standard library that returned tuples have - been modified to return :class:`namedtuple` instances. For example, + been modified to return :func:`namedtuple` instances. For example, the :meth:`Decimal.as_tuple` method now returns a named tuple with :attr:`sign`, :attr:`digits`, and :attr:`exponent` fields. @@ -2289,7 +2290,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger; :issue:`1861`.) * The :mod:`select` module now has wrapper functions - for the Linux :c:func:`epoll` and BSD :c:func:`kqueue` system calls. + for the Linux :c:func:`!epoll` and BSD :c:func:`!kqueue` system calls. :meth:`modify` method was added to the existing :class:`poll` objects; ``pollobj.modify(fd, eventmask)`` takes a file descriptor or file object and an event mask, modifying the recorded event mask @@ -2328,7 +2329,7 @@ changes, or look through the Subversion logs for all the details. one for reading and one for writing. The writable descriptor will be passed to :func:`set_wakeup_fd`, and the readable descriptor will be added to the list of descriptors monitored by the event loop via - :c:func:`select` or :c:func:`poll`. + :c:func:`!select` or :c:func:`!poll`. On receiving a signal, a byte will be written and the main event loop will be woken up, avoiding the need to poll. @@ -2465,7 +2466,7 @@ changes, or look through the Subversion logs for all the details. (All changes contributed by Lars Gustäbel). * An optional ``timeout`` parameter was added to the - :class:`telnetlib.Telnet` class constructor, specifying a timeout + :class:`!telnetlib.Telnet` class constructor, specifying a timeout measured in seconds. (Added by Facundo Batista.) * The :class:`tempfile.NamedTemporaryFile` class usually deletes @@ -2982,7 +2983,7 @@ Changes to Python's build process and to the C API include: * Python now must be compiled with C89 compilers (after 19 years!). This means that the Python source tree has dropped its - own implementations of :c:func:`memmove` and :c:func:`strerror`, which + own implementations of :c:func:`!memmove` and :c:func:`!strerror`, which are in the C89 standard library. * Python 2.6 can be built with Microsoft Visual Studio 2008 (version @@ -3012,11 +3013,11 @@ Changes to Python's build process and to the C API include: bug occurred if one thread closed a file object while another thread was reading from or writing to the object. In 2.6 file objects have a reference count, manipulated by the - :c:func:`PyFile_IncUseCount` and :c:func:`PyFile_DecUseCount` + :c:func:`!PyFile_IncUseCount` and :c:func:`!PyFile_DecUseCount` functions. File objects can't be closed unless the reference count - is zero. :c:func:`PyFile_IncUseCount` should be called while the GIL + is zero. :c:func:`!PyFile_IncUseCount` should be called while the GIL is still held, before carrying out an I/O operation using the - ``FILE *`` pointer, and :c:func:`PyFile_DecUseCount` should be called + ``FILE *`` pointer, and :c:func:`!PyFile_DecUseCount` should be called immediately after the GIL is re-acquired. (Contributed by Antoine Pitrou and Gregory P. Smith.) @@ -3060,9 +3061,9 @@ Changes to Python's build process and to the C API include: * Some macros were renamed in both 3.0 and 2.6 to make it clearer that they are macros, - not functions. :c:macro:`Py_Size()` became :c:macro:`Py_SIZE()`, - :c:macro:`Py_Type()` became :c:macro:`Py_TYPE()`, and - :c:macro:`Py_Refcnt()` became :c:macro:`Py_REFCNT()`. + not functions. :c:macro:`!Py_Size()` became :c:macro:`Py_SIZE()`, + :c:macro:`!Py_Type()` became :c:macro:`Py_TYPE()`, and + :c:macro:`!Py_Refcnt()` became :c:macro:`Py_REFCNT()`. The mixed-case macros are still available in Python 2.6 for backward compatibility. (:issue:`1629`) @@ -3135,7 +3136,7 @@ Port-Specific Changes: Windows registry reflection for 32-bit processes running on 64-bit systems. (:issue:`1753245`) -* The :mod:`msilib` module's :class:`Record` object +* The :mod:`!msilib` module's :class:`Record` object gained :meth:`GetInteger` and :meth:`GetString` methods that return field values as an integer or a string. (Contributed by Floris Bruynooghe; :issue:`2125`.) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 36afcb163f1afc..cf6d26859bb6a2 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -355,7 +355,7 @@ added as a more powerful replacement for the This means Python now supports three different modules for parsing command-line arguments: :mod:`getopt`, :mod:`optparse`, and :mod:`argparse`. The :mod:`getopt` module closely resembles the C -library's :c:func:`getopt` function, so it remains useful if you're writing a +library's :c:func:`!getopt` function, so it remains useful if you're writing a Python prototype that will eventually be rewritten in C. :mod:`optparse` becomes redundant, but there are no plans to remove it because there are many scripts still using it, and there's no @@ -858,9 +858,11 @@ Some smaller changes made to the core Python language are: .. XXX bytearray doesn't seem to be documented -* When using ``@classmethod`` and ``@staticmethod`` to wrap +* When using :class:`@classmethod ` and + :class:`@staticmethod ` to wrap methods as class or static methods, the wrapper object now - exposes the wrapped function as their :attr:`__func__` attribute. + exposes the wrapped function as their :attr:`~method.__func__` + attribute. (Contributed by Amaury Forgeot d'Arc, after a suggestion by George Sakkis; :issue:`5982`.) @@ -930,8 +932,8 @@ Optimizations Several performance enhancements have been added: * A new opcode was added to perform the initial setup for - :keyword:`with` statements, looking up the :meth:`__enter__` and - :meth:`__exit__` methods. (Contributed by Benjamin Peterson.) + :keyword:`with` statements, looking up the :meth:`~object.__enter__` and + :meth:`~object.__exit__` methods. (Contributed by Benjamin Peterson.) * The garbage collector now performs better for one common usage pattern: when many objects are being allocated without deallocating @@ -1429,7 +1431,7 @@ changes, or look through the Subversion logs for all the details. become very large. (Contributed by Charles Cazabon; :issue:`6963`.) -* The :mod:`nntplib` module now supports IPv6 addresses. +* The :mod:`!nntplib` module now supports IPv6 addresses. (Contributed by Derek Morr; :issue:`1664`.) * New functions: the :mod:`os` module wraps the following POSIX system @@ -1556,9 +1558,9 @@ changes, or look through the Subversion logs for all the details. :issue:`8484`.) The version of OpenSSL being used is now available as the module - attributes :data:`ssl.OPENSSL_VERSION` (a string), - :data:`ssl.OPENSSL_VERSION_INFO` (a 5-tuple), and - :data:`ssl.OPENSSL_VERSION_NUMBER` (an integer). (Added by Antoine + attributes :const:`ssl.OPENSSL_VERSION` (a string), + :const:`ssl.OPENSSL_VERSION_INFO` (a 5-tuple), and + :const:`ssl.OPENSSL_VERSION_NUMBER` (an integer). (Added by Antoine Pitrou; :issue:`8321`.) * The :mod:`struct` module will no longer silently ignore overflow @@ -2152,16 +2154,16 @@ Changes to Python's build process and to the C API include: * New function: stemming from the rewrite of string-to-float conversion, a new :c:func:`PyOS_string_to_double` function was added. The old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions are now deprecated. -* New function: :c:func:`PySys_SetArgvEx` sets the value of +* New function: :c:func:`!PySys_SetArgvEx` sets the value of ``sys.argv`` and can optionally update ``sys.path`` to include the directory containing the script named by ``sys.argv[0]`` depending on the value of an *updatepath* parameter. This function was added to close a security hole for applications - that embed Python. The old function, :c:func:`PySys_SetArgv`, would + that embed Python. The old function, :c:func:`!PySys_SetArgv`, would always update ``sys.path``, and sometimes it would add the current directory. This meant that, if you ran an application embedding Python in a directory controlled by someone else, attackers could @@ -2169,8 +2171,8 @@ Changes to Python's build process and to the C API include: :file:`os.py`) that your application would then import and run. If you maintain a C/C++ application that embeds Python, check - whether you're calling :c:func:`PySys_SetArgv` and carefully consider - whether the application should be using :c:func:`PySys_SetArgvEx` + whether you're calling :c:func:`!PySys_SetArgv` and carefully consider + whether the application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. Security issue reported as `CVE-2008-5983 @@ -2195,13 +2197,13 @@ Changes to Python's build process and to the C API include: .. XXX these macros don't seem to be described in the c-api docs. -* Removed function: :c:macro:`PyEval_CallObject` is now only available +* Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be deleted by now. (Removed by Antoine Pitrou; :issue:`8276`.) -* New format codes: the :c:func:`PyFormat_FromString`, - :c:func:`PyFormat_FromStringV`, and :c:func:`PyErr_Format` functions now +* New format codes: the :c:func:`!PyString_FromFormat`, + :c:func:`!PyString_FromFormatV`, and :c:func:`PyErr_Format` functions now accept ``%lld`` and ``%llu`` format codes for displaying C's :c:expr:`long long` types. (Contributed by Mark Dickinson; :issue:`7228`.) @@ -2231,7 +2233,7 @@ Changes to Python's build process and to the C API include: * When using the :c:type:`PyMemberDef` structure to define attributes of a type, Python will no longer let you try to delete or set a - :const:`T_STRING_INPLACE` attribute. + :c:macro:`T_STRING_INPLACE` attribute. .. rev 79644 @@ -2287,10 +2289,10 @@ object, and then get the ``void *`` pointer, which will usually point to an array of pointers to the module's various API functions. There is an existing data type already used for this, -:c:type:`PyCObject`, but it doesn't provide type safety. Evil code +:c:type:`!PyCObject`, but it doesn't provide type safety. Evil code written in pure Python could cause a segmentation fault by taking a -:c:type:`PyCObject` from module A and somehow substituting it for the -:c:type:`PyCObject` in module B. Capsules know their own name, +:c:type:`!PyCObject` from module A and somehow substituting it for the +:c:type:`!PyCObject` in module B. Capsules know their own name, and getting the pointer requires providing the name: .. code-block:: c @@ -2310,10 +2312,10 @@ detect the mismatched name and return false. Refer to :ref:`using-capsules` for more information on using these objects. Python 2.7 now uses capsules internally to provide various -extension-module APIs, but the :c:func:`PyCObject_AsVoidPtr` was +extension-module APIs, but the :c:func:`!PyCObject_AsVoidPtr` was modified to handle capsules, preserving compile-time compatibility -with the :c:type:`CObject` interface. Use of -:c:func:`PyCObject_AsVoidPtr` will signal a +with the :c:type:`!PyCObject` interface. Use of +:c:func:`!PyCObject_AsVoidPtr` will signal a :exc:`PendingDeprecationWarning`, which is silent by default. Implemented in Python 3.1 and backported to 2.7 by Larry Hastings; @@ -2368,7 +2370,7 @@ Port-Specific Changes: Mac OS X installation and a user-installed copy of the same version. (Changed by Ronald Oussoren; :issue:`4865`.) - .. versionchanged:: 2.7.13 + .. versionchanged:: 2.7.13 As of 2.7.13, this change was removed. ``/Library/Python/2.7/site-packages``, the site-packages directory @@ -2383,8 +2385,8 @@ Port-Specific Changes: Mac OS X Port-Specific Changes: FreeBSD ----------------------------------- -* FreeBSD 7.1's :const:`SO_SETFIB` constant, used with - :func:`~socket.getsockopt`/:func:`~socket.setsockopt` to select an +* FreeBSD 7.1's :const:`SO_SETFIB` constant, used with the :func:`~socket.socket` methods + :func:`~socket.socket.getsockopt`/:func:`~socket.socket.setsockopt` to select an alternate routing table, is now available in the :mod:`socket` module. (Added by Kyle VanderBeek; :issue:`8235`.) @@ -2404,7 +2406,7 @@ Other Changes and Fixes :issue:`5464`.) * When importing a module from a :file:`.pyc` or :file:`.pyo` file - with an existing :file:`.py` counterpart, the :attr:`co_filename` + with an existing :file:`.py` counterpart, the :attr:`~codeobject.co_filename` attributes of the resulting code objects are overwritten when the original filename is obsolete. This can happen if the file has been renamed, moved, or is accessed through different paths. (Patch by @@ -2449,13 +2451,13 @@ that may require changes to your code: (Changed by Eric Smith; :issue:`5920`.) * Because of an optimization for the :keyword:`with` statement, the special - methods :meth:`__enter__` and :meth:`__exit__` must belong to the object's + methods :meth:`~object.__enter__` and :meth:`~object.__exit__` must belong to the object's type, and cannot be directly attached to the object's instance. This affects new-style classes (derived from :class:`object`) and C extension types. (:issue:`6101`.) * Due to a bug in Python 2.6, the *exc_value* parameter to - :meth:`__exit__` methods was often the string representation of the + :meth:`~object.__exit__` methods was often the string representation of the exception, not an instance. This was fixed in 2.7, so *exc_value* will be an instance as expected. (Fixed by Florent Xicluna; :issue:`7853`.) @@ -2540,16 +2542,16 @@ For C extensions: instead of triggering a :exc:`DeprecationWarning` (:issue:`5080`). * Use the new :c:func:`PyOS_string_to_double` function instead of the old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions, + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions, which are now deprecated. For applications that embed Python: -* The :c:func:`PySys_SetArgvEx` function was added, letting +* The :c:func:`!PySys_SetArgvEx` function was added, letting applications close a security hole when the existing - :c:func:`PySys_SetArgv` function was used. Check whether you're - calling :c:func:`PySys_SetArgv` and carefully consider whether the - application should be using :c:func:`PySys_SetArgvEx` with + :c:func:`!PySys_SetArgv` function was used. Check whether you're + calling :c:func:`!PySys_SetArgv` and carefully consider whether the + application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. .. ====================================================================== diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index f9ac13036cbc8d..b0c2529e780213 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -789,7 +789,7 @@ Operators And Special Methods :attr:`__doc__`, :attr:`__globals__`, :attr:`~definition.__name__`, respectively. -* :meth:`__nonzero__` is now :meth:`__bool__`. +* :meth:`!__nonzero__` is now :meth:`~object.__bool__`. Builtins -------- @@ -865,8 +865,8 @@ to the C API. * No more C API support for restricted execution. -* :c:func:`PyNumber_Coerce`, :c:func:`PyNumber_CoerceEx`, - :c:func:`PyMember_Get`, and :c:func:`PyMember_Set` C APIs are removed. +* :c:func:`!PyNumber_Coerce`, :c:func:`!PyNumber_CoerceEx`, + :c:func:`!PyMember_Get`, and :c:func:`!PyMember_Set` C APIs are removed. * New C API :c:func:`PyImport_ImportModuleNoBlock`, works like :c:func:`PyImport_ImportModule` but won't block on the import lock @@ -875,7 +875,7 @@ to the C API. * Renamed the boolean conversion C-level slot and method: ``nb_nonzero`` is now ``nb_bool``. -* Removed :c:macro:`METH_OLDARGS` and :c:macro:`WITH_CYCLE_GC` from the C API. +* Removed :c:macro:`!METH_OLDARGS` and :c:macro:`!WITH_CYCLE_GC` from the C API. .. ====================================================================== @@ -911,7 +911,7 @@ best strategy is the following: tests still pass. 3. Run the ``2to3`` source-to-source translator over your source code - tree. (See :ref:`2to3-reference` for more on this tool.) Run the + tree. Run the result of the translation under Python 3.0. Manually fix up any remaining issues, fixing problems until all tests pass again. diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index fba8816bb243a3..e237179f4b1829 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -370,14 +370,14 @@ New, Improved, and Deprecated Modules * The :mod:`io` module has three new constants for the :meth:`seek` method :data:`SEEK_SET`, :data:`SEEK_CUR`, and :data:`SEEK_END`. -* The :attr:`sys.version_info` tuple is now a named tuple:: +* The :data:`sys.version_info` tuple is now a named tuple:: >>> sys.version_info sys.version_info(major=3, minor=1, micro=0, releaselevel='alpha', serial=2) (Contributed by Ross Light; :issue:`4285`.) -* The :mod:`nntplib` and :mod:`imaplib` modules now support IPv6. +* The :mod:`!nntplib` and :mod:`imaplib` modules now support IPv6. (Contributed by Derek Morr; :issue:`1655` and :issue:`1664`.) @@ -486,7 +486,7 @@ Changes to Python's build process and to the C API include: Apart from the performance improvements this change should be invisible to end users, with one exception: for testing and debugging purposes there's a - new :attr:`sys.int_info` that provides information about the + new :data:`sys.int_info` that provides information about the internal format, giving the number of bits per digit and the size in bytes of the C type used to store each digit:: @@ -501,16 +501,16 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson and Lisandro Dalcrin; :issue:`5175`.) -* Deprecated :c:func:`PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. +* Deprecated :c:func:`!PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. (Contributed by Mark Dickinson; :issue:`4910`.) * Added a new :c:func:`PyOS_string_to_double` function to replace the - deprecated functions :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof`. + deprecated functions :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof`. (Contributed by Mark Dickinson; :issue:`5914`.) -* Added :c:type:`PyCapsule` as a replacement for the :c:type:`PyCObject` API. +* Added :c:type:`PyCapsule` as a replacement for the :c:type:`!PyCObject` API. The principal difference is that the new type has a well defined interface for passing typing safety information and a less complicated signature for calling a destructor. The old type had a problematic API and is now diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 661eeaedbfc0d0..2da90b7ed55744 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -221,116 +221,116 @@ have been incorporated. Some of the most notable ones are as follows: * Missing ``:`` before blocks: - .. code-block:: python + .. code-block:: python - >>> if rocket.position > event_horizon - File "", line 1 - if rocket.position > event_horizon - ^ - SyntaxError: expected ':' + >>> if rocket.position > event_horizon + File "", line 1 + if rocket.position > event_horizon + ^ + SyntaxError: expected ':' - (Contributed by Pablo Galindo in :issue:`42997`.) + (Contributed by Pablo Galindo in :issue:`42997`.) * Unparenthesised tuples in comprehensions targets: - .. code-block:: python + .. code-block:: python - >>> {x,y for x,y in zip('abcd', '1234')} - File "", line 1 - {x,y for x,y in zip('abcd', '1234')} - ^ - SyntaxError: did you forget parentheses around the comprehension target? + >>> {x,y for x,y in zip('abcd', '1234')} + File "", line 1 + {x,y for x,y in zip('abcd', '1234')} + ^ + SyntaxError: did you forget parentheses around the comprehension target? - (Contributed by Pablo Galindo in :issue:`43017`.) + (Contributed by Pablo Galindo in :issue:`43017`.) * Missing commas in collection literals and between expressions: - .. code-block:: python + .. code-block:: python - >>> items = { - ... x: 1, - ... y: 2 - ... z: 3, - File "", line 3 - y: 2 - ^ - SyntaxError: invalid syntax. Perhaps you forgot a comma? + >>> items = { + ... x: 1, + ... y: 2 + ... z: 3, + File "", line 3 + y: 2 + ^ + SyntaxError: invalid syntax. Perhaps you forgot a comma? - (Contributed by Pablo Galindo in :issue:`43822`.) + (Contributed by Pablo Galindo in :issue:`43822`.) * Multiple Exception types without parentheses: - .. code-block:: python + .. code-block:: python - >>> try: - ... build_dyson_sphere() - ... except NotEnoughScienceError, NotEnoughResourcesError: - File "", line 3 - except NotEnoughScienceError, NotEnoughResourcesError: - ^ - SyntaxError: multiple exception types must be parenthesized + >>> try: + ... build_dyson_sphere() + ... except NotEnoughScienceError, NotEnoughResourcesError: + File "", line 3 + except NotEnoughScienceError, NotEnoughResourcesError: + ^ + SyntaxError: multiple exception types must be parenthesized - (Contributed by Pablo Galindo in :issue:`43149`.) + (Contributed by Pablo Galindo in :issue:`43149`.) * Missing ``:`` and values in dictionary literals: - .. code-block:: python + .. code-block:: python - >>> values = { - ... x: 1, - ... y: 2, - ... z: - ... } - File "", line 4 - z: - ^ - SyntaxError: expression expected after dictionary key and ':' + >>> values = { + ... x: 1, + ... y: 2, + ... z: + ... } + File "", line 4 + z: + ^ + SyntaxError: expression expected after dictionary key and ':' - >>> values = {x:1, y:2, z w:3} - File "", line 1 - values = {x:1, y:2, z w:3} - ^ - SyntaxError: ':' expected after dictionary key + >>> values = {x:1, y:2, z w:3} + File "", line 1 + values = {x:1, y:2, z w:3} + ^ + SyntaxError: ':' expected after dictionary key - (Contributed by Pablo Galindo in :issue:`43823`.) + (Contributed by Pablo Galindo in :issue:`43823`.) * ``try`` blocks without ``except`` or ``finally`` blocks: - .. code-block:: python + .. code-block:: python - >>> try: - ... x = 2 - ... something = 3 - File "", line 3 - something = 3 - ^^^^^^^^^ - SyntaxError: expected 'except' or 'finally' block + >>> try: + ... x = 2 + ... something = 3 + File "", line 3 + something = 3 + ^^^^^^^^^ + SyntaxError: expected 'except' or 'finally' block - (Contributed by Pablo Galindo in :issue:`44305`.) + (Contributed by Pablo Galindo in :issue:`44305`.) * Usage of ``=`` instead of ``==`` in comparisons: - .. code-block:: python + .. code-block:: python - >>> if rocket.position = event_horizon: - File "", line 1 - if rocket.position = event_horizon: - ^ - SyntaxError: cannot assign to attribute here. Maybe you meant '==' instead of '='? + >>> if rocket.position = event_horizon: + File "", line 1 + if rocket.position = event_horizon: + ^ + SyntaxError: cannot assign to attribute here. Maybe you meant '==' instead of '='? - (Contributed by Pablo Galindo in :issue:`43797`.) + (Contributed by Pablo Galindo in :issue:`43797`.) * Usage of ``*`` in f-strings: - .. code-block:: python + .. code-block:: python - >>> f"Black holes {*all_black_holes} and revelations" - File "", line 1 - (*all_black_holes) - ^ - SyntaxError: f-string: cannot use starred expression here + >>> f"Black holes {*all_black_holes} and revelations" + File "", line 1 + (*all_black_holes) + ^ + SyntaxError: f-string: cannot use starred expression here - (Contributed by Pablo Galindo in :issue:`41064`.) + (Contributed by Pablo Galindo in :issue:`41064`.) IndentationErrors ~~~~~~~~~~~~~~~~~ @@ -365,10 +365,10 @@ raised from: (Contributed by Pablo Galindo in :issue:`38530`.) - .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error - which can happen if some other custom error display function is used. This is a common - scenario in some REPLs like IPython. +.. warning:: + Notice this won't work if :c:func:`PyErr_Display` is not called to display the error + which can happen if some other custom error display function is used. This is a common + scenario in some REPLs like IPython. NameErrors ~~~~~~~~~~ @@ -387,10 +387,10 @@ was raised from: (Contributed by Pablo Galindo in :issue:`38530`.) - .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error, - which can happen if some other custom error display function is used. This is a common - scenario in some REPLs like IPython. +.. warning:: + Notice this won't work if :c:func:`PyErr_Display` is not called to display the error, + which can happen if some other custom error display function is used. This is a common + scenario in some REPLs like IPython. PEP 626: Precise line numbers for debugging and other tools @@ -399,9 +399,11 @@ PEP 626: Precise line numbers for debugging and other tools PEP 626 brings more precise and reliable line numbers for debugging, profiling and coverage tools. Tracing events, with the correct line number, are generated for all lines of code executed and only for lines of code that are executed. -The ``f_lineno`` attribute of frame objects will always contain the expected line number. +The :attr:`~frame.f_lineno` attribute of frame objects will always contain the +expected line number. -The ``co_lnotab`` attribute of code objects is deprecated and will be removed in 3.12. +The :attr:`~codeobject.co_lnotab` attribute of code objects is deprecated and +will be removed in 3.12. Code that needs to convert from offset to line number should use the new ``co_lines()`` method instead. PEP 634: Structural Pattern Matching @@ -433,16 +435,16 @@ A match statement takes an expression and compares its value to successive patterns given as one or more case blocks. Specifically, pattern matching operates by: - 1. using data with type and shape (the ``subject``) - 2. evaluating the ``subject`` in the ``match`` statement - 3. comparing the subject with each pattern in a ``case`` statement - from top to bottom until a match is confirmed. - 4. executing the action associated with the pattern of the confirmed - match - 5. If an exact match is not confirmed, the last case, a wildcard ``_``, - if provided, will be used as the matching case. If an exact match is - not confirmed and a wildcard case does not exist, the entire match - block is a no-op. +1. using data with type and shape (the ``subject``) +2. evaluating the ``subject`` in the ``match`` statement +3. comparing the subject with each pattern in a ``case`` statement + from top to bottom until a match is confirmed. +4. executing the action associated with the pattern of the confirmed + match +5. If an exact match is not confirmed, the last case, a wildcard ``_``, + if provided, will be used as the matching case. If an exact match is + not confirmed and a wildcard case does not exist, the entire match + block is a no-op. Declarative approach ~~~~~~~~~~~~~~~~~~~~ @@ -878,7 +880,7 @@ Other Language Changes (Contributed by Raymond Hettinger in :issue:`43475`.) * A :exc:`SyntaxError` (instead of a :exc:`NameError`) will be raised when deleting - the :const:`__debug__` constant. (Contributed by Dong-hee Na in :issue:`45000`.) + the :const:`__debug__` constant. (Contributed by Donghee Na in :issue:`45000`.) * :exc:`SyntaxError` exceptions now have ``end_lineno`` and ``end_offset`` attributes. They will be ``None`` if not determined. @@ -887,7 +889,7 @@ Other Language Changes New Modules =========== -* None yet. +* None. Improved Modules @@ -1253,9 +1255,9 @@ descriptors without copying between kernel address space and user address space, where one of the file descriptors must refer to a pipe. (Contributed by Pablo Galindo in :issue:`41625`.) -Add :data:`~os.O_EVTONLY`, :data:`~os.O_FSYNC`, :data:`~os.O_SYMLINK` -and :data:`~os.O_NOFOLLOW_ANY` for macOS. -(Contributed by Dong-hee Na in :issue:`43106`.) +Add :const:`~os.O_EVTONLY`, :const:`~os.O_FSYNC`, :const:`~os.O_SYMLINK` +and :const:`~os.O_NOFOLLOW_ANY` for macOS. +(Contributed by Donghee Na in :issue:`43106`.) os.path ------- @@ -1319,7 +1321,7 @@ objects in the tree returned by :func:`pyclbr.readline` and shelve ------ -The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default +The :mod:`shelve` module now uses :const:`pickle.DEFAULT_PROTOCOL` by default instead of :mod:`pickle` protocol ``3`` when creating shelves. (Contributed by Zackery Spytz in :issue:`34204`.) @@ -1356,7 +1358,7 @@ The ssl module requires OpenSSL 1.1.1 or newer. (Contributed by Christian Heimes in :pep:`644` and :issue:`43669`.) The ssl module has preliminary support for OpenSSL 3.0.0 and new option -:data:`~ssl.OP_IGNORE_UNEXPECTED_EOF`. +:const:`~ssl.OP_IGNORE_UNEXPECTED_EOF`. (Contributed by Christian Heimes in :issue:`38820`, :issue:`43794`, :issue:`43788`, :issue:`43791`, :issue:`43799`, :issue:`43920`, :issue:`43789`, and :issue:`43811`.) @@ -1387,7 +1389,7 @@ Add a *timeout* parameter to the :func:`ssl.get_server_certificate` function. The ssl module uses heap-types and multi-phase initialization. (Contributed by Christian Heimes in :issue:`42333`.) -A new verify flag :data:`~ssl.VERIFY_X509_PARTIAL_CHAIN` has been added. +A new verify flag :const:`~ssl.VERIFY_X509_PARTIAL_CHAIN` has been added. (Contributed by l0x in :issue:`40849`.) sqlite3 @@ -1413,7 +1415,7 @@ _thread ------- :func:`_thread.interrupt_main` now takes an optional signal number to -simulate (the default is still :data:`signal.SIGINT`). +simulate (the default is still :const:`signal.SIGINT`). (Contributed by Antoine Pitrou in :issue:`43356`.) threading @@ -1508,7 +1510,7 @@ query parameter separators in :func:`urllib.parse.parse_qs` and :func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with newer W3C recommendations, this has been changed to allow only a single separator key, with ``&`` as the default. This change also affects -:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +:func:`!cgi.parse` and :func:`!cgi.parse_multipart` as they use the affected functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) @@ -1582,7 +1584,7 @@ Optimizations * The following built-in functions now support the faster :pep:`590` vectorcall calling convention: :func:`map`, :func:`filter`, :func:`reversed`, :func:`bool` and :func:`float`. - (Contributed by Dong-hee Na and Jeroen Demeyer in :issue:`43575`, :issue:`43287`, :issue:`41922`, :issue:`41873` and :issue:`41870`.) + (Contributed by Donghee Na and Jeroen Demeyer in :issue:`43575`, :issue:`43287`, :issue:`41922`, :issue:`41873` and :issue:`41870`.) * :class:`BZ2File` performance is improved by removing internal ``RLock``. This makes :class:`BZ2File` thread unsafe in the face of multiple simultaneous @@ -1757,8 +1759,8 @@ Deprecated * :data:`~ssl.PROTOCOL_SSLv2`, :data:`~ssl.PROTOCOL_SSLv3`, :data:`~ssl.PROTOCOL_SSLv23`, :data:`~ssl.PROTOCOL_TLSv1`, :data:`~ssl.PROTOCOL_TLSv1_1`, :data:`~ssl.PROTOCOL_TLSv1_2`, and - :data:`~ssl.PROTOCOL_TLS` are deprecated in favor of - :data:`~ssl.PROTOCOL_TLS_CLIENT` and :data:`~ssl.PROTOCOL_TLS_SERVER` + :const:`~ssl.PROTOCOL_TLS` are deprecated in favor of + :const:`~ssl.PROTOCOL_TLS_CLIENT` and :const:`~ssl.PROTOCOL_TLS_SERVER` * :func:`~ssl.wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` @@ -1817,10 +1819,10 @@ Removed scheduled to be removed in Python 3.6, but such removals were delayed until after Python 2.7 EOL. Existing users should copy whatever classes they use into their code. - (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) + (Contributed by Donghee Na and Terry J. Reedy in :issue:`42299`.) -* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless - now due to the _warnings module was converted to a builtin module in 2.6. +* Removed the :c:func:`!PyModule_GetWarningsModule` function that was useless + now due to the :mod:`!_warnings` module was converted to a builtin module in 2.6. (Contributed by Hai Shi in :issue:`42599`.) * Remove deprecated aliases to :ref:`collections-abstract-base-classes` from @@ -1959,11 +1961,11 @@ Changes in the C API source_buf = PyBytes_AsString(source_bytes_object); code = Py_CompileString(source_buf, filename, Py_file_input); - * For ``FrameObject`` objects, the ``f_lasti`` member now represents a wordcode + * For ``FrameObject`` objects, the :attr:`~frame.f_lasti` member now represents a wordcode offset instead of a simple offset into the bytecode string. This means that this number needs to be multiplied by 2 to be used with APIs that expect a byte offset instead (like :c:func:`PyCode_Addr2Line` for example). Notice as well that the - ``f_lasti`` member of ``FrameObject`` objects is not considered stable: please + :attr:`!f_lasti` member of ``FrameObject`` objects is not considered stable: please use :c:func:`PyFrame_GetLineNumber` instead. CPython bytecode changes @@ -2124,11 +2126,11 @@ New Features These functions allow to activate, deactivate and query the state of the garbage collector from C code without having to import the :mod:`gc` module. -* Add a new :c:data:`Py_TPFLAGS_DISALLOW_INSTANTIATION` type flag to disallow +* Add a new :c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` type flag to disallow creating type instances. (Contributed by Victor Stinner in :issue:`43916`.) -* Add a new :c:data:`Py_TPFLAGS_IMMUTABLETYPE` type flag for creating immutable +* Add a new :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` type flag for creating immutable type objects: type attributes cannot be set nor deleted. (Contributed by Victor Stinner and Erlend E. Aasland in :issue:`43908`.) @@ -2187,9 +2189,9 @@ Porting to Python 3.10 been included directly, consider including ``Python.h`` instead. (Contributed by Nicholas Sim in :issue:`35134`.) -* Use the :c:data:`Py_TPFLAGS_IMMUTABLETYPE` type flag to create immutable type - objects. Do not rely on :c:data:`Py_TPFLAGS_HEAPTYPE` to decide if a type - object is mutable or not; check if :c:data:`Py_TPFLAGS_IMMUTABLETYPE` is set +* Use the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` type flag to create immutable type + objects. Do not rely on :c:macro:`Py_TPFLAGS_HEAPTYPE` to decide if a type + object is mutable or not; check if :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` is set instead. (Contributed by Victor Stinner and Erlend E. Aasland in :issue:`43908`.) @@ -2211,16 +2213,16 @@ Removed * Removed ``Py_UNICODE_str*`` functions manipulating ``Py_UNICODE*`` strings. (Contributed by Inada Naoki in :issue:`41123`.) - * ``Py_UNICODE_strlen``: use :c:func:`PyUnicode_GetLength` or - :c:macro:`PyUnicode_GET_LENGTH` - * ``Py_UNICODE_strcat``: use :c:func:`PyUnicode_CopyCharacters` or - :c:func:`PyUnicode_FromFormat` - * ``Py_UNICODE_strcpy``, ``Py_UNICODE_strncpy``: use - :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` - * ``Py_UNICODE_strcmp``: use :c:func:`PyUnicode_Compare` - * ``Py_UNICODE_strncmp``: use :c:func:`PyUnicode_Tailmatch` - * ``Py_UNICODE_strchr``, ``Py_UNICODE_strrchr``: use - :c:func:`PyUnicode_FindChar` + * ``Py_UNICODE_strlen``: use :c:func:`PyUnicode_GetLength` or + :c:macro:`PyUnicode_GET_LENGTH` + * ``Py_UNICODE_strcat``: use :c:func:`PyUnicode_CopyCharacters` or + :c:func:`PyUnicode_FromFormat` + * ``Py_UNICODE_strcpy``, ``Py_UNICODE_strncpy``: use + :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` + * ``Py_UNICODE_strcmp``: use :c:func:`PyUnicode_Compare` + * ``Py_UNICODE_strncmp``: use :c:func:`PyUnicode_Tailmatch` + * ``Py_UNICODE_strchr``, ``Py_UNICODE_strrchr``: use + :c:func:`PyUnicode_FindChar` * Removed ``PyUnicode_GetMax()``. Please migrate to new (:pep:`393`) APIs. (Contributed by Inada Naoki in :issue:`41103`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7a479c6e56a9f0..8db133b90a7a4b 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -45,7 +45,7 @@ when researching a change. This article explains the new features in Python 3.11, compared to 3.10. - +Python 3.11 was released on October 24, 2022. For full details, see the :ref:`changelog `. @@ -218,7 +218,7 @@ Windows ``py.exe`` launcher improvements The copy of the :ref:`launcher` included with Python 3.11 has been significantly updated. It now supports company/tag syntax as defined in :pep:`514` using the -``-V:/`` argument instead of the limited ``-.``. +:samp:`-V:{}/{}` argument instead of the limited :samp:`-{}.{}`. This allows launching distributions other than ``PythonCore``, the one hosted on `python.org `_. @@ -227,8 +227,8 @@ installs will be searched. For example, ``-V:OtherPython/`` will select the "best" tag registered for ``OtherPython``, while ``-V:3.11`` or ``-V:/3.11`` will select the "best" distribution with tag ``3.11``. -When using the legacy ``-``, ``-.``, -``--`` or ``-.-`` arguments, +When using the legacy :samp:`-{}`, :samp:`-{}.{}`, +:samp:`-{}-{}` or :samp:`-{}.{}-{}` arguments, all existing behaviour should be preserved from past versions, and only releases from ``PythonCore`` will be selected. However, the ``-64`` suffix now implies "not 32-bit" (not necessarily x86-64), @@ -459,6 +459,10 @@ Other Language Changes :class:`collections.OrderedDict`, :class:`collections.deque`, :class:`weakref.WeakSet`, and :class:`datetime.tzinfo` now copies and pickles instance attributes implemented as :term:`slots <__slots__>`. + This change has an unintended side effect: It trips up a small minority + of existing Python projects not expecting :meth:`object.__getstate__` to + exist. See the later comments on :gh:`70766` for discussions of what + workarounds such code may need. (Contributed by Serhiy Storchaka in :issue:`26579`.) .. _whatsnew311-pythonsafepath: @@ -495,7 +499,7 @@ Other CPython Implementation Changes * The special methods :meth:`~object.__complex__` for :class:`complex` and :meth:`~object.__bytes__` for :class:`bytes` are implemented to support the :class:`typing.SupportsComplex` and :class:`typing.SupportsBytes` protocols. - (Contributed by Mark Dickinson and Dong-hee Na in :issue:`24234`.) + (Contributed by Mark Dickinson and Donghee Na in :issue:`24234`.) * ``siphash13`` is added as a new internal hashing algorithm. It has similar security properties as ``siphash24``, @@ -640,7 +644,7 @@ dataclasses datetime -------- -* Add :attr:`datetime.UTC`, a convenience alias for +* Add :const:`datetime.UTC`, a convenience alias for :attr:`datetime.timezone.utc`. (Contributed by Kabir Kwatra in :gh:`91973`.) * :meth:`datetime.date.fromisoformat`, :meth:`datetime.time.fromisoformat` and @@ -690,7 +694,7 @@ enum * Added the :func:`~enum.global_enum` enum decorator, which adjusts :meth:`~object.__repr__` and :meth:`~object.__str__` to show values as members of their module rather than the enum class. - For example, ``'re.ASCII'`` for the :data:`~re.ASCII` member + For example, ``'re.ASCII'`` for the :const:`~re.ASCII` member of :class:`re.RegexFlag` rather than ``'RegexFlag.ASCII'``. * Enhanced :class:`~enum.Flag` to support @@ -893,7 +897,7 @@ os * On Windows, :func:`os.urandom` now uses ``BCryptGenRandom()``, instead of ``CryptGenRandom()`` which is deprecated. - (Contributed by Dong-hee Na in :issue:`44611`.) + (Contributed by Donghee Na in :issue:`44611`.) .. _whatsnew311-pathlib: @@ -1063,8 +1067,8 @@ threading * On Unix, if the ``sem_clockwait()`` function is available in the C library (glibc 2.30 and newer), the :meth:`threading.Lock.acquire` method now uses - the monotonic clock (:data:`time.CLOCK_MONOTONIC`) for the timeout, rather - than using the system clock (:data:`time.CLOCK_REALTIME`), to not be affected + the monotonic clock (:const:`time.CLOCK_MONOTONIC`) for the timeout, rather + than using the system clock (:const:`time.CLOCK_REALTIME`), to not be affected by system clock changes. (Contributed by Victor Stinner in :issue:`41710`.) @@ -1085,7 +1089,7 @@ time `_ which has a resolution of 100 nanoseconds (10\ :sup:`-7` seconds). Previously, it had a resolution of 1 millisecond (10\ :sup:`-3` seconds). - (Contributed by Benjamin SzÅ‘ke, Dong-hee Na, Eryk Sun and Victor Stinner in :issue:`21302` and :issue:`45429`.) + (Contributed by Benjamin SzÅ‘ke, Donghee Na, Eryk Sun and Victor Stinner in :issue:`21302` and :issue:`45429`.) .. _whatsnew311-tkinter: @@ -1301,7 +1305,7 @@ This section covers specific optimizations independent of the * :func:`unicodedata.normalize` now normalizes pure-ASCII strings in constant time. - (Contributed by Dong-hee Na in :issue:`44987`.) + (Contributed by Donghee Na in :issue:`44987`.) .. _whatsnew311-faster-cpython: @@ -1448,7 +1452,7 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) | | | | (up to) | | +===============+====================+=======================================================+===================+===================+ | Binary | ``x + x`` | Binary add, multiply and subtract for common types | 10% | Mark Shannon, | -| operations | | such as :class:`int`, :class:`float` and :class:`str` | | Dong-hee Na, | +| operations | | such as :class:`int`, :class:`float` and :class:`str` | | Donghee Na, | | | ``x - x`` | take custom fast paths for their underlying types. | | Brandt Bucher, | | | | | | Dennis Sweeney | | | ``x * x`` | | | | @@ -1731,13 +1735,13 @@ Modules slated for removal in Python 3.13: +---------------------+---------------------+---------------------+---------------------+---------------------+ - | :mod:`aifc` | :mod:`chunk` | :mod:`msilib` | :mod:`pipes` | :mod:`telnetlib` | + | :mod:`!aifc` | :mod:`!chunk` | :mod:`!msilib` | :mod:`!pipes` | :mod:`!telnetlib` | +---------------------+---------------------+---------------------+---------------------+---------------------+ - | :mod:`audioop` | :mod:`crypt` | :mod:`nis` | :mod:`sndhdr` | :mod:`uu` | + | :mod:`!audioop` | :mod:`!crypt` | :mod:`!nis` | :mod:`!sndhdr` | :mod:`!uu` | +---------------------+---------------------+---------------------+---------------------+---------------------+ - | :mod:`cgi` | :mod:`imghdr` | :mod:`nntplib` | :mod:`spwd` | :mod:`xdrlib` | + | :mod:`!cgi` | :mod:`!imghdr` | :mod:`!nntplib` | :mod:`!spwd` | :mod:`!xdrlib` | +---------------------+---------------------+---------------------+---------------------+---------------------+ - | :mod:`cgitb` | :mod:`mailcap` | :mod:`ossaudiodev` | :mod:`sunau` | | + | :mod:`!cgitb` | :mod:`!mailcap` | :mod:`!ossaudiodev` | :mod:`!sunau` | | +---------------------+---------------------+---------------------+---------------------+---------------------+ (Contributed by Brett Cannon in :issue:`47061` and Victor Stinner in @@ -1748,7 +1752,7 @@ Modules warnings have now been updated to note they will be removed in Python 3.12. (Contributed by Hugo van Kemenade in :issue:`47022`.) -* The :mod:`lib2to3` package and :ref:`2to3 <2to3-reference>` tool +* The :mod:`!lib2to3` package and ``2to3`` tool are now deprecated and may not be able to parse Python 3.10 or newer. See :pep:`617`, introducing the new PEG parser, for details. (Contributed by Victor Stinner in :issue:`40360`.) @@ -1794,7 +1798,7 @@ Standard Library * :func:`importlib.resources.path` * The :func:`locale.getdefaultlocale` function is deprecated and will be - removed in Python 3.13. Use :func:`locale.setlocale`, + removed in Python 3.15. Use :func:`locale.setlocale`, :func:`locale.getpreferredencoding(False) ` and :func:`locale.getlocale` functions instead. (Contributed by Victor Stinner in :gh:`90817`.) @@ -1812,12 +1816,12 @@ Standard Library (Contributed by Serhiy Storchaka in :gh:`91760`.) * In the :mod:`re` module, the :func:`!re.template` function - and the corresponding :data:`!re.TEMPLATE` and :data:`!re.T` flags + and the corresponding :const:`!re.TEMPLATE` and :const:`!re.T` flags are deprecated, as they were undocumented and lacked an obvious purpose. They will be removed in Python 3.13. (Contributed by Serhiy Storchaka and Miro HronÄok in :gh:`92728`.) -* :func:`turtle.settiltangle` has been deprecated since Python 3.1; +* :func:`!turtle.settiltangle` has been deprecated since Python 3.1; it now emits a deprecation warning and will be removed in Python 3.13. Use :func:`turtle.tiltangle` instead (it was earlier incorrectly marked as deprecated, and its docstring is now corrected). @@ -1835,7 +1839,7 @@ Standard Library * :class:`!webbrowser.MacOSX` is deprecated and will be removed in Python 3.13. It is untested, undocumented, and not used by :mod:`webbrowser` itself. - (Contributed by Dong-hee Na in :issue:`42255`.) + (Contributed by Donghee Na in :issue:`42255`.) * The behavior of returning a value from a :class:`~unittest.TestCase` and :class:`~unittest.IsolatedAsyncioTestCase` test methods (other than the @@ -1856,6 +1860,10 @@ Standard Library (Contributed by Erlend E. Aasland in :issue:`5846`.) +* :meth:`~!unittest.TestProgram.usageExit` is marked deprecated, to be removed + in 3.13. + (Contributed by Carlos Damázio in :gh:`67048`.) + .. _whatsnew311-pending-removal: .. _whatsnew311-python-api-pending-removal: @@ -1976,7 +1984,7 @@ Removed C APIs are :ref:`listed separately `. :meth:`!NullTranslations.set_output_charset` methods, and the *codeset* parameter of :func:`!translation` and :func:`!install`, since they are only used for the :func:`!l*gettext` functions. - (Contributed by Dong-hee Na and Serhiy Storchaka in :issue:`44235`.) + (Contributed by Donghee Na and Serhiy Storchaka in :issue:`44235`.) * Removed from the :mod:`inspect` module: @@ -2001,7 +2009,7 @@ Removed C APIs are :ref:`listed separately `. * Removed the :class:`!MailmanProxy` class in the :mod:`smtpd` module, as it is unusable without the external :mod:`!mailman` package. - (Contributed by Dong-hee Na in :issue:`35800`.) + (Contributed by Donghee Na in :issue:`35800`.) * Removed the deprecated :meth:`!split` method of :class:`!_tkinter.TkappType`. (Contributed by Erlend E. Aasland in :issue:`38371`.) @@ -2143,7 +2151,7 @@ Build Changes * CPython can now be built with the `ThinLTO `_ option via passing ``thin`` to :option:`--with-lto`, i.e. ``--with-lto=thin``. - (Contributed by Dong-hee Na and Brett Holman in :issue:`44340`.) + (Contributed by Donghee Na and Brett Holman in :issue:`44340`.) * Freelists for object structs can now be disabled. A new :program:`configure` option :option:`--without-freelists` can be used to disable all freelists @@ -2212,7 +2220,7 @@ New Features * :c:func:`PyBuffer_SizeFromFormat` * :c:func:`PyBuffer_ToContiguous` * :c:func:`PyBuffer_FromContiguous` - * :c:func:`PyBuffer_CopyData` + * :c:func:`PyObject_CopyData` * :c:func:`PyBuffer_IsContiguous` * :c:func:`PyBuffer_FillContiguousStrides` * :c:func:`PyBuffer_FillInfo` @@ -2223,7 +2231,7 @@ New Features (Contributed by Christian Heimes in :issue:`45459`.) -* Added the :c:data:`PyType_GetModuleByDef` function, used to get the module +* Added the :c:func:`PyType_GetModuleByDef` function, used to get the module in which a method was defined, in cases where this information is not available directly (via :c:type:`PyCMethod`). (Contributed by Petr Viktorin in :issue:`46613`.) @@ -2343,11 +2351,11 @@ Porting to Python 3.11 #endif * The :c:func:`PyType_Ready` function now raises an error if a type is defined - with the :const:`Py_TPFLAGS_HAVE_GC` flag set but has no traverse function + with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set but has no traverse function (:c:member:`PyTypeObject.tp_traverse`). (Contributed by Victor Stinner in :issue:`44263`.) -* Heap types with the :const:`Py_TPFLAGS_IMMUTABLETYPE` flag can now inherit +* Heap types with the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag can now inherit the :pep:`590` vectorcall protocol. Previously, this was only possible for :ref:`static types `. (Contributed by Erlend E. Aasland in :issue:`43908`) @@ -2450,11 +2458,12 @@ Porting to Python 3.11 * ``f_valuestack``: removed. The Python frame object is now created lazily. A side effect is that the - ``f_back`` member must not be accessed directly, since its value is now also + :attr:`~frame.f_back` member must not be accessed directly, + since its value is now also computed lazily. The :c:func:`PyFrame_GetBack` function must be called instead. - Debuggers that accessed the ``f_locals`` directly *must* call + Debuggers that accessed the :attr:`~frame.f_locals` directly *must* call :c:func:`PyFrame_GetLocals` instead. They no longer need to call :c:func:`PyFrame_FastToLocalsWithError` or :c:func:`PyFrame_LocalsToFast`, in fact they should not call those functions. The necessary updating of the @@ -2558,18 +2567,18 @@ Deprecated * Deprecate the following functions to configure the Python initialization: - * :c:func:`PySys_AddWarnOptionUnicode` - * :c:func:`PySys_AddWarnOption` - * :c:func:`PySys_AddXOption` - * :c:func:`PySys_HasWarnOptions` - * :c:func:`PySys_SetArgvEx` - * :c:func:`PySys_SetArgv` - * :c:func:`PySys_SetPath` - * :c:func:`Py_SetPath` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`Py_SetStandardStreamEncoding` - * :c:func:`_Py_SetProgramFullPath` + * :c:func:`!PySys_AddWarnOptionUnicode` + * :c:func:`!PySys_AddWarnOption` + * :c:func:`!PySys_AddXOption` + * :c:func:`!PySys_HasWarnOptions` + * :c:func:`!PySys_SetArgvEx` + * :c:func:`!PySys_SetArgv` + * :c:func:`!PySys_SetPath` + * :c:func:`!Py_SetPath` + * :c:func:`!Py_SetProgramName` + * :c:func:`!Py_SetPythonHome` + * :c:func:`!Py_SetStandardStreamEncoding` + * :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). @@ -2587,22 +2596,22 @@ Pending Removal in Python 3.12 The following C APIs have been deprecated in earlier Python releases, and will be removed in Python 3.12. -* :c:func:`PyUnicode_AS_DATA` -* :c:func:`PyUnicode_AS_UNICODE` -* :c:func:`PyUnicode_AsUnicodeAndSize` -* :c:func:`PyUnicode_AsUnicode` -* :c:func:`PyUnicode_FromUnicode` -* :c:func:`PyUnicode_GET_DATA_SIZE` -* :c:func:`PyUnicode_GET_SIZE` -* :c:func:`PyUnicode_GetSize` +* :c:func:`!PyUnicode_AS_DATA` +* :c:func:`!PyUnicode_AS_UNICODE` +* :c:func:`!PyUnicode_AsUnicodeAndSize` +* :c:func:`!PyUnicode_AsUnicode` +* :c:func:`!PyUnicode_FromUnicode` +* :c:func:`!PyUnicode_GET_DATA_SIZE` +* :c:func:`!PyUnicode_GET_SIZE` +* :c:func:`!PyUnicode_GetSize` * :c:func:`PyUnicode_IS_COMPACT` * :c:func:`PyUnicode_IS_READY` * :c:func:`PyUnicode_READY` -* :c:func:`Py_UNICODE_WSTR_LENGTH` -* :c:func:`_PyUnicode_AsUnicode` -* :c:macro:`PyUnicode_WCHAR_KIND` +* :c:func:`!PyUnicode_WSTR_LENGTH` +* :c:func:`!_PyUnicode_AsUnicode` +* :c:macro:`!PyUnicode_WCHAR_KIND` * :c:type:`PyUnicodeObject` -* :c:func:`PyUnicode_InternImmortal()` +* :c:func:`!PyUnicode_InternImmortal` .. _whatsnew311-c-api-removed: @@ -2610,7 +2619,7 @@ and will be removed in Python 3.12. Removed ------- -* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been +* :c:func:`!PyFrame_BlockSetup` and :c:func:`!PyFrame_BlockPop` have been removed. (Contributed by Mark Shannon in :issue:`40222`.) @@ -2646,7 +2655,7 @@ Removed * :c:func:`PyMarshal_WriteObjectToString` * the ``Py_MARSHAL_VERSION`` macro - These are not part of the :ref:`limited API `. + These are not part of the :ref:`limited API `. (Contributed by Victor Stinner in :issue:`45474`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3e55b3fa0f4734..8551b35438e2c3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -3,8 +3,7 @@ What's New In Python 3.12 **************************** -:Release: |release| -:Date: |today| +:Editor: Adam Turner .. Rules for maintenance: @@ -47,15 +46,12 @@ researching a change. This article explains the new features in Python 3.12, compared to 3.11. - +Python 3.12 was released on October 2, 2023. For full details, see the :ref:`changelog `. -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.12 moves towards release, - so it's worth checking back even after reading earlier versions. +.. seealso:: + :pep:`693` -- Python 3.12 Release Schedule Summary -- Release highlights ============================= @@ -63,29 +59,375 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.12. Brevity is key. +Python 3.12 is the latest stable release of the Python programming language, +with a mix of changes to the language and the standard library. +The library changes focus on cleaning up deprecated APIs, usability, and correctness. +Of note, the :mod:`!distutils` package has been removed from the standard library. +Filesystem support in :mod:`os` and :mod:`pathlib` has seen a number of improvements, +and several modules have better performance. + +The language changes focus on usability, +as :term:`f-strings ` have had many limitations removed +and 'Did you mean ...' suggestions continue to improve. +The new :ref:`type parameter syntax ` +and :keyword:`type` statement improve ergonomics for using :term:`generic types +` and :term:`type aliases ` with static type checkers. + +This article doesn't attempt to provide a complete specification of all new features, +but instead gives a convenient overview. +For full details, you should refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +If you want to understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. + +-------------- .. PEP-sized items next. -New typing features: +New syntax features: + +* :ref:`PEP 695 `, type parameter syntax and the :keyword:`type` statement + +New grammar features: + +* :ref:`PEP 701 `, :term:`f-strings ` in the grammar + +Interpreter improvements: + +* :ref:`PEP 684 `, a unique per-interpreter :term:`GIL + ` +* :ref:`PEP 669 `, low impact monitoring +* `Improved 'Did you mean ...' suggestions `_ + for :exc:`NameError`, :exc:`ImportError`, and :exc:`SyntaxError` exceptions + +Python data model improvements: + +* :ref:`PEP 688 `, using the :ref:`buffer protocol + ` from Python + +Significant improvements in the standard library: + +* The :class:`pathlib.Path` class now supports subclassing +* The :mod:`os` module received several improvements for Windows support +* A :ref:`command-line interface ` has been added to the + :mod:`sqlite3` module +* :func:`isinstance` checks against :func:`runtime-checkable protocols + ` enjoy a speed up of between two and 20 times +* The :mod:`asyncio` package has had a number of performance improvements, + with some benchmarks showing a 75% speed up. +* A :ref:`command-line interface ` has been added to the + :mod:`uuid` module +* Due to the changes in :ref:`PEP 701 `, + producing tokens via the :mod:`tokenize` module is up to 64% faster. + +Security improvements: + +* Replace the builtin :mod:`hashlib` implementations of + SHA1, SHA3, SHA2-384, SHA2-512, and MD5 with formally verified code from the + `HACL* `__ project. + These builtin implementations remain as fallbacks that are only used when + OpenSSL does not provide them. + +C API improvements: + +* :ref:`PEP 697 `, unstable C API tier +* :ref:`PEP 683 `, immortal objects -* :pep:`688`: Making the buffer protocol accessible in Python +CPython implementation improvements: -* :ref:`whatsnew312-pep692` +* :ref:`PEP 709 `, comprehension inlining +* :ref:`CPython support ` for the Linux ``perf`` profiler +* Implement stack overflow protection on supported platforms -* :pep:`698`: Override Decorator for Static Typing +New typing features: + +* :ref:`PEP 692 `, using :class:`~typing.TypedDict` to + annotate :term:`**kwargs ` +* :ref:`PEP 698 `, :func:`typing.override` decorator Important deprecations, removals or restrictions: -* :pep:`623`: Remove wstr from Unicode +* :pep:`623`: Remove ``wstr`` from Unicode objects in Python's C API, + reducing the size of every :class:`str` object by at least 8 bytes. + +* :pep:`632`: Remove the :mod:`!distutils` package. + See `the migration guide `_ + for advice replacing the APIs it provided. + The third-party `Setuptools `__ + package continues to provide :mod:`!distutils`, + if you still require it in Python 3.12 and beyond. + +* :gh:`95299`: Do not pre-install ``setuptools`` in virtual environments + created with :mod:`venv`. + This means that ``distutils``, ``setuptools``, ``pkg_resources``, + and ``easy_install`` will no longer available by default; to access these + run ``pip install setuptools`` in the :ref:`activated ` + virtual environment. + +* The :mod:`!asynchat`, :mod:`!asyncore`, and :mod:`!imp` modules have been + removed, along with several :class:`unittest.TestCase` + `method aliases `_. + + +New Features +============ + +.. _whatsnew312-pep695: + +PEP 695: Type Parameter Syntax +------------------------------ + +Generic classes and functions under :pep:`484` were declared using a verbose syntax +that left the scope of type parameters unclear and required explicit declarations of +variance. + +:pep:`695` introduces a new, more compact and explicit way to create +:ref:`generic classes ` and :ref:`functions `:: + + def max[T](args: Iterable[T]) -> T: + ... + + class list[T]: + def __getitem__(self, index: int, /) -> T: + ... + + def append(self, element: T) -> None: + ... + +In addition, the PEP introduces a new way to declare :ref:`type aliases ` +using the :keyword:`type` statement, which creates an instance of +:class:`~typing.TypeAliasType`:: + + type Point = tuple[float, float] + +Type aliases can also be :ref:`generic `:: + + type Point[T] = tuple[T, T] + +The new syntax allows declaring :class:`~typing.TypeVarTuple` +and :class:`~typing.ParamSpec` parameters, as well as :class:`~typing.TypeVar` +parameters with bounds or constraints:: + + type IntFunc[**P] = Callable[P, int] # ParamSpec + type LabeledTuple[*Ts] = tuple[str, *Ts] # TypeVarTuple + type HashableSequence[T: Hashable] = Sequence[T] # TypeVar with bound + type IntOrStrSequence[T: (int, str)] = Sequence[T] # TypeVar with constraints + +The value of type aliases and the bound and constraints of type variables +created through this syntax are evaluated only on demand (see +:ref:`lazy evaluation `). This means type aliases are able to +refer to other types defined later in the file. + +Type parameters declared through a type parameter list are visible within the +scope of the declaration and any nested scopes, but not in the outer scope. For +example, they can be used in the type annotations for the methods of a generic +class or in the class body. However, they cannot be used in the module scope after +the class is defined. See :ref:`type-params` for a detailed description of the +runtime semantics of type parameters. + +In order to support these scoping semantics, a new kind of scope is introduced, +the :ref:`annotation scope `. Annotation scopes behave for the +most part like function scopes, but interact differently with enclosing class scopes. +In Python 3.13, :term:`annotations ` will also be evaluated in +annotation scopes. + +See :pep:`695` for more details. + +(PEP written by Eric Traut. Implementation by Jelle Zijlstra, Eric Traut, +and others in :gh:`103764`.) + +.. _whatsnew312-pep701: + +PEP 701: Syntactic formalization of f-strings +--------------------------------------------- + +:pep:`701` lifts some restrictions on the usage of :term:`f-strings `. +Expression components inside f-strings can now be any valid Python expression, +including strings reusing the same quote as the containing f-string, +multi-line expressions, comments, backslashes, and unicode escape sequences. +Let's cover these in detail: + +* Quote reuse: in Python 3.11, reusing the same quotes as the enclosing f-string + raises a :exc:`SyntaxError`, forcing the user to either use other available + quotes (like using double quotes or triple quotes if the f-string uses single + quotes). In Python 3.12, you can now do things like this: + + >>> songs = ['Take me back to Eden', 'Alkaline', 'Ascensionism'] + >>> f"This is the playlist: {", ".join(songs)}" + 'This is the playlist: Take me back to Eden, Alkaline, Ascensionism' + + Note that before this change there was no explicit limit in how f-strings can + be nested, but the fact that string quotes cannot be reused inside the + expression component of f-strings made it impossible to nest f-strings + arbitrarily. In fact, this is the most nested f-string that could be written: + + >>> f"""{f'''{f'{f"{1+1}"}'}'''}""" + '2' + + As now f-strings can contain any valid Python expression inside expression + components, it is now possible to nest f-strings arbitrarily: + + >>> f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" + '2' + +* Multi-line expressions and comments: In Python 3.11, f-string expressions + must be defined in a single line, even if the expression within the f-string + could normally span multiple lines + (like literal lists being defined over multiple lines), + making them harder to read. In Python 3.12 you can now define f-strings + spanning multiple lines, and add inline comments: + + >>> f"This is the playlist: {", ".join([ + ... 'Take me back to Eden', # My, my, those eyes like fire + ... 'Alkaline', # Not acid nor alkaline + ... 'Ascensionism' # Take to the broken skies at last + ... ])}" + 'This is the playlist: Take me back to Eden, Alkaline, Ascensionism' + +* Backslashes and unicode characters: before Python 3.12 f-string expressions + couldn't contain any ``\`` character. This also affected unicode :ref:`escape + sequences ` (such as ``\N{snowman}``) as these contain + the ``\N`` part that previously could not be part of expression components of + f-strings. Now, you can define expressions like this: + + >>> print(f"This is the playlist: {"\n".join(songs)}") + This is the playlist: Take me back to Eden + Alkaline + Ascensionism + >>> print(f"This is the playlist: {"\N{BLACK HEART SUIT}".join(songs)}") + This is the playlist: Take me back to Eden♥Alkaline♥Ascensionism + +See :pep:`701` for more details. + +As a positive side-effect of how this feature has been implemented (by parsing f-strings +with :pep:`the PEG parser <617>`), now error messages for f-strings are more precise +and include the exact location of the error. For example, in Python 3.11, the following +f-string raises a :exc:`SyntaxError`: + +.. code-block:: python + + >>> my_string = f"{x z y}" + f"{1 + 1}" + File "", line 1 + (x z y) + ^^^ + SyntaxError: f-string: invalid syntax. Perhaps you forgot a comma? + +but the error message doesn't include the exact location of the error within the line and +also has the expression artificially surrounded by parentheses. In Python 3.12, as f-strings +are parsed with the PEG parser, error messages can be more precise and show the entire line: + +.. code-block:: python + + >>> my_string = f"{x z y}" + f"{1 + 1}" + File "", line 1 + my_string = f"{x z y}" + f"{1 + 1}" + ^^^ + SyntaxError: invalid syntax. Perhaps you forgot a comma? + +(Contributed by Pablo Galindo, Batuhan Taskaya, Lysandros Nikolaou, Cristián +Maureira-Fredes and Marta Gómez in :gh:`102856`. PEP written by Pablo Galindo, +Batuhan Taskaya, Lysandros Nikolaou and Marta Gómez). -* :pep:`632`: Remove the ``distutils`` package +.. _whatsnew312-pep684: + +PEP 684: A Per-Interpreter GIL +------------------------------ + +:pep:`684` introduces a per-interpreter :term:`GIL `, +so that sub-interpreters may now be created with a unique GIL per interpreter. +This allows Python programs to take full advantage of multiple CPU +cores. This is currently only available through the C-API, +though a Python API is :pep:`anticipated for 3.13 <554>`. + +Use the new :c:func:`Py_NewInterpreterFromConfig` function to +create an interpreter with its own GIL:: + + PyInterpreterConfig config = { + .check_multi_interp_extensions = 1, + .gil = PyInterpreterConfig_OWN_GIL, + }; + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + return -1; + } + /* The new interpreter is now active in the current thread. */ + +For further examples how to use the C-API for sub-interpreters with a +per-interpreter GIL, see :source:`Modules/_xxsubinterpretersmodule.c`. + +(Contributed by Eric Snow in :gh:`104210`, etc.) + +.. _whatsnew312-pep669: + +PEP 669: Low impact monitoring for CPython +------------------------------------------ + +:pep:`669` defines a new :mod:`API ` for profilers, +debuggers, and other tools to monitor events in CPython. +It covers a wide range of events, including calls, +returns, lines, exceptions, jumps, and more. +This means that you only pay for what you use, providing support +for near-zero overhead debuggers and coverage tools. +See :mod:`sys.monitoring` for details. + +(Contributed by Mark Shannon in :gh:`103082`.) + +.. _whatsnew312-pep688: + +PEP 688: Making the buffer protocol accessible in Python +-------------------------------------------------------- + +:pep:`688` introduces a way to use the :ref:`buffer protocol ` +from Python code. Classes that implement the :meth:`~object.__buffer__` method +are now usable as buffer types. + +The new :class:`collections.abc.Buffer` ABC provides a standard +way to represent buffer objects, for example in type annotations. +The new :class:`inspect.BufferFlags` enum represents the flags that +can be used to customize buffer creation. +(Contributed by Jelle Zijlstra in :gh:`102500`.) + +.. _whatsnew312-pep709: + +PEP 709: Comprehension inlining +------------------------------- + +Dictionary, list, and set comprehensions are now inlined, rather than creating a +new single-use function object for each execution of the comprehension. This +speeds up execution of a comprehension by up to two times. +See :pep:`709` for further details. + +Comprehension iteration variables remain isolated and don't overwrite a +variable of the same name in the outer scope, nor are they visible after the +comprehension. Inlining does result in a few visible behavior changes: + +* There is no longer a separate frame for the comprehension in tracebacks, + and tracing/profiling no longer shows the comprehension as a function call. +* The :mod:`symtable` module will no longer produce child symbol tables for each + comprehension; instead, the comprehension's locals will be included in the + parent function's symbol table. +* Calling :func:`locals` inside a comprehension now includes variables + from outside the comprehension, and no longer includes the synthetic ``.0`` + variable for the comprehension "argument". +* A comprehension iterating directly over ``locals()`` (e.g. ``[k for k in + locals()]``) may see "RuntimeError: dictionary changed size during iteration" + when run under tracing (e.g. code coverage measurement). This is the same + behavior already seen in e.g. ``for k in locals():``. To avoid the error, first + create a list of keys to iterate over: ``keys = list(locals()); [k for k in + keys]``. + +(Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`.) Improved Error Messages -======================= +----------------------- * Modules from the standard library are now potentially suggested as part of the error messages displayed by the interpreter when a :exc:`NameError` is - raised to the top level. Contributed by Pablo Galindo in :gh:`98254`. + raised to the top level. (Contributed by Pablo Galindo in :gh:`98254`.) >>> sys.version_info Traceback (most recent call last): @@ -96,7 +438,7 @@ Improved Error Messages Now if a :exc:`NameError` is raised in a method and the instance has an attribute that's exactly equal to the name in the exception, the suggestion will include ``self.`` instead of the closest match in the method - scope. Contributed by Pablo Galindo in :gh:`99139`. + scope. (Contributed by Pablo Galindo in :gh:`99139`.) >>> class A: ... def __init__(self): @@ -104,7 +446,7 @@ Improved Error Messages ... ... def foo(self): ... somethin = blech - + ... >>> A().foo() Traceback (most recent call last): File "", line 1 @@ -112,9 +454,8 @@ Improved Error Messages ^^^^^ NameError: name 'blech' is not defined. Did you mean: 'self.blech'? - * Improve the :exc:`SyntaxError` error message when the user types ``import x - from y`` instead of ``from y import x``. Contributed by Pablo Galindo in :gh:`98931`. + from y`` instead of ``from y import x``. (Contributed by Pablo Galindo in :gh:`98931`.) >>> import a.y.z from b.y.z Traceback (most recent call last): @@ -125,7 +466,7 @@ Improved Error Messages * :exc:`ImportError` exceptions raised from failed ``from import `` statements now include suggestions for the value of ```` based on the - available names in ````. Contributed by Pablo Galindo in :gh:`91058`. + available names in ````. (Contributed by Pablo Galindo in :gh:`91058`.) >>> from collections import chainmap Traceback (most recent call last): @@ -133,67 +474,10 @@ Improved Error Messages ImportError: cannot import name 'chainmap' from 'collections'. Did you mean: 'ChainMap'? -New Features -============ - -* Add :ref:`perf_profiling` through the new - environment variable :envvar:`PYTHONPERFSUPPORT`, - the new command-line option :option:`-X perf <-X>`, - as well as the new :func:`sys.activate_stack_trampoline`, - :func:`sys.deactivate_stack_trampoline`, - and :func:`sys.is_stack_trampoline_active` APIs. - (Design by Pablo Galindo. Contributed by Pablo Galindo and Christian Heimes - with contributions from Gregory P. Smith [Google] and Mark Shannon - in :gh:`96123`.) -* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, - have a new a *filter* argument that allows limiting tar features than may be - surprising or dangerous, such as creating files outside the destination - directory. - See :ref:`tarfile-extraction-filter` for details. - In Python 3.14, the default will switch to ``'data'``. - (Contributed by Petr Viktorin in :pep:`706`.) - -.. _whatsnew312-pep709: - -PEP 709: Comprehension inlining -------------------------------- - -Dictionary, list, and set comprehensions are now inlined, rather than creating a -new single-use function object for each execution of the comprehension. This -speeds up execution of a comprehension by up to 2x. - -Comprehension iteration variables remain isolated; they don't overwrite a -variable of the same name in the outer scope, nor are they visible after the -comprehension. This isolation is now maintained via stack/locals manipulation, -not via separate function scope. - -Inlining does result in a few visible behavior changes: - -* There is no longer a separate frame for the comprehension in tracebacks, - and tracing/profiling no longer shows the comprehension as a function call. -* Calling :func:`locals` inside a comprehension now includes variables - from outside the comprehension, and no longer includes the synthetic ``.0`` - variable for the comprehension "argument". - -Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`. - -PEP 688: Making the buffer protocol accessible in Python --------------------------------------------------------- - -:pep:`688` introduces a way to use the :ref:`buffer protocol ` -from Python code. Classes that implement the :meth:`~object.__buffer__` method -are now usable as buffer types. - -The new :class:`collections.abc.Buffer` ABC provides a standard -way to represent buffer objects, for example in type annotations. -The new :class:`inspect.BufferFlags` enum represents the flags that -can be used to customize buffer creation. -(Contributed by Jelle Zijlstra in :gh:`102500`.) - New Features Related to Type Hints ================================== -This section covers major changes affecting :pep:`484` type hints and +This section covers major changes affecting :pep:`type hints <484>` and the :mod:`typing` module. .. _whatsnew312-pep692: @@ -205,7 +489,7 @@ Typing ``**kwargs`` in a function signature as introduced by :pep:`484` allowed for valid annotations only in cases where all of the ``**kwargs`` were of the same type. -This PEP specifies a more precise way of typing ``**kwargs`` by relying on +:pep:`692` specifies a more precise way of typing ``**kwargs`` by relying on typed dictionaries:: from typing import TypedDict, Unpack @@ -218,37 +502,52 @@ typed dictionaries:: See :pep:`692` for more details. -(PEP written by Franek Magiera) +(Contributed by Franek Magiera in :gh:`103629`.) -Other Language Changes -====================== +.. _whatsnew312-pep698: -* :class:`types.MappingProxyType` instances are now hashable if the underlying - mapping is hashable. - (Contributed by Serhiy Storchaka in :gh:`87995`.) +PEP 698: Override Decorator for Static Typing +--------------------------------------------- -* :class:`memoryview` now supports the half-float type (the "e" format code). - (Contributed by Dong-hee Na and Antoine Pitrou in :gh:`90751`.) +A new decorator :func:`typing.override` has been added to the :mod:`typing` +module. It indicates to type checkers that the method is intended to override +a method in a superclass. This allows type checkers to catch mistakes where +a method that is intended to override something in a base class +does not in fact do so. -* The parser now raises :exc:`SyntaxError` when parsing source code containing - null bytes. (Contributed by Pablo Galindo in :gh:`96670`.) +Example:: -* :func:`ast.parse` now raises :exc:`SyntaxError` instead of :exc:`ValueError` - when parsing source code containing null bytes. (Contributed by Pablo Galindo - in :gh:`96670`.) + from typing import override -* The Garbage Collector now runs only on the eval breaker mechanism of the - Python bytecode evaluation loop instead of object allocations. The GC can - also run when :c:func:`PyErr_CheckSignals` is called so C extensions that - need to run for a long time without executing any Python code also have a - chance to execute the GC periodically. (Contributed by Pablo Galindo in - :gh:`97922`.) + class Base: + def get_color(self) -> str: + return "blue" + + class GoodChild(Base): + @override # ok: overrides Base.get_color + def get_color(self) -> str: + return "yellow" + + class BadChild(Base): + @override # type checker error: does not override Base.get_color + def get_colour(self) -> str: + return "red" + +See :pep:`698` for more details. + +(Contributed by Steven Troxler in :gh:`101561`.) + +Other Language Changes +====================== + +* The parser now raises :exc:`SyntaxError` when parsing source code containing + null bytes. (Contributed by Pablo Galindo in :gh:`96670`.) * A backslash-character pair that is not a valid escape sequence now generates a :exc:`SyntaxWarning`, instead of :exc:`DeprecationWarning`. For example, ``re.compile("\d+\.\d+")`` now emits a :exc:`SyntaxWarning` - (``"\d"`` is an invalid escape sequence), use raw strings for regular - expression: ``re.compile(r"\d+\.\d+")``. + (``"\d"`` is an invalid escape sequence, use raw strings for regular + expression: ``re.compile(r"\d+\.\d+")``). In a future Python version, :exc:`SyntaxError` will eventually be raised, instead of :exc:`SyntaxWarning`. (Contributed by Victor Stinner in :gh:`98401`.) @@ -259,10 +558,6 @@ Other Language Changes In a future Python version they will be eventually a :exc:`SyntaxError`. (Contributed by Victor Stinner in :gh:`98401`.) -* All builtin and extension callables expecting boolean parameters now accept - arguments of any type instead of just :class:`bool` and :class:`int`. - (Contributed by Serhiy Storchaka in :gh:`60203`.) - * Variables used in the target part of comprehensions that are not stored to can now be used in assignment expressions (``:=``). For example, in ``[(b := 1) for a, b.prop in some_iter]``, the assignment to @@ -270,10 +565,7 @@ Other Language Changes part of comprehensions (like ``a``) is still disallowed, as per :pep:`572`. (Contributed by Nikita Sobolev in :gh:`100581`.) -* :class:`slice` objects are now hashable, allowing them to be used as dict keys and - set items. (Contributed by Will Bradshaw and Furkan Onder in :gh:`101264`.) - -* Exceptions raised in a typeobject's ``__set_name__`` method are no longer +* Exceptions raised in a class or type's ``__set_name__`` method are no longer wrapped by a :exc:`RuntimeError`. Context information is added to the exception as a :pep:`678` note. (Contributed by Irit Katriel in :gh:`77757`.) @@ -282,11 +574,58 @@ Other Language Changes :exc:`ExceptionGroup`. Also changed in version 3.11.4. (Contributed by Irit Katriel in :gh:`103590`.) +* The Garbage Collector now runs only on the eval breaker mechanism of the + Python bytecode evaluation loop instead of object allocations. The GC can + also run when :c:func:`PyErr_CheckSignals` is called so C extensions that + need to run for a long time without executing any Python code also have a + chance to execute the GC periodically. (Contributed by Pablo Galindo in + :gh:`97922`.) + +* All builtin and extension callables expecting boolean parameters now accept + arguments of any type instead of just :class:`bool` and :class:`int`. + (Contributed by Serhiy Storchaka in :gh:`60203`.) + +* :class:`memoryview` now supports the half-float type (the "e" format code). + (Contributed by Donghee Na and Antoine Pitrou in :gh:`90751`.) + +* :class:`slice` objects are now hashable, allowing them to be used as dict keys and + set items. (Contributed by Will Bradshaw, Furkan Onder, and Raymond Hettinger in :gh:`101264`.) + +* :func:`sum` now uses Neumaier summation to improve accuracy and commutativity + when summing floats or mixed ints and floats. + (Contributed by Raymond Hettinger in :gh:`100425`.) + +* :func:`ast.parse` now raises :exc:`SyntaxError` instead of :exc:`ValueError` + when parsing source code containing null bytes. (Contributed by Pablo Galindo + in :gh:`96670`.) + +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile extraction filters ` for details. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) + +* :class:`types.MappingProxyType` instances are now hashable if the underlying + mapping is hashable. + (Contributed by Serhiy Storchaka in :gh:`87995`.) + +* Add :ref:`support for the perf profiler ` through the new + environment variable :envvar:`PYTHONPERFSUPPORT` + and command-line option :option:`-X perf <-X>`, + as well as the new :func:`sys.activate_stack_trampoline`, + :func:`sys.deactivate_stack_trampoline`, + and :func:`sys.is_stack_trampoline_active` functions. + (Design by Pablo Galindo. Contributed by Pablo Galindo and Christian Heimes + with contributions from Gregory P. Smith [Google] and Mark Shannon + in :gh:`96123`.) + New Modules =========== -* None yet. +* None. Improved Modules @@ -306,29 +645,20 @@ asyncio writing to sockets and uses :meth:`~socket.socket.sendmsg` if the platform supports it. (Contributed by Kumar Aditya in :gh:`91166`.) -* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` +* Add :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` functions to allow opting an event loop in to eager task execution, making some use-cases 2x to 5x faster. - (Contributed by Jacob Bower & Itamar O in :gh:`102853`, :gh:`104140`, and :gh:`104138`) + (Contributed by Jacob Bower & Itamar Oren in :gh:`102853`, :gh:`104140`, and :gh:`104138`) -* On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default +* On Linux, :mod:`asyncio` uses :class:`asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of - :class:`~asyncio.ThreadedChildWatcher`. + :class:`asyncio.ThreadedChildWatcher`. (Contributed by Kumar Aditya in :gh:`98024`.) -* The child watcher classes :class:`~asyncio.MultiLoopChildWatcher`, - :class:`~asyncio.FastChildWatcher`, :class:`~asyncio.AbstractChildWatcher` - and :class:`~asyncio.SafeChildWatcher` are deprecated and - will be removed in Python 3.14. It is recommended to not manually - configure a child watcher as the event loop now uses the best available - child watcher for each platform (:class:`~asyncio.PidfdChildWatcher` - if supported and :class:`~asyncio.ThreadedChildWatcher` otherwise). - (Contributed by Kumar Aditya in :gh:`94597`.) - -* :func:`asyncio.set_child_watcher`, :func:`asyncio.get_child_watcher`, - :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` and - :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated - and will be removed in Python 3.14. +* The event loop now uses the best available child watcher for each platform + (:class:`asyncio.PidfdChildWatcher` if supported and + :class:`asyncio.ThreadedChildWatcher` otherwise), so manually + configuring a child watcher is not recommended. (Contributed by Kumar Aditya in :gh:`94597`.) * Add *loop_factory* parameter to :func:`asyncio.run` to allow specifying @@ -336,7 +666,7 @@ asyncio (Contributed by Kumar Aditya in :gh:`99388`.) * Add C implementation of :func:`asyncio.current_task` for 4x-6x speedup. - (Contributed by Itamar Ostricher and Pranav Thulasiram Bhat in :gh:`100344`.) + (Contributed by Itamar Oren and Pranav Thulasiram Bhat in :gh:`100344`.) * :func:`asyncio.iscoroutine` now returns ``False`` for generators as :mod:`asyncio` does not support legacy generator-based coroutines. @@ -349,15 +679,45 @@ asyncio calendar -------- -* Add enums :data:`~calendar.Month` and :data:`~calendar.Day`. +* Add enums :data:`calendar.Month` and :data:`calendar.Day` + defining months of the year and days of the week. (Contributed by Prince Roshan in :gh:`103636`.) csv --- -* Add :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` flags to +* Add :const:`csv.QUOTE_NOTNULL` and :const:`csv.QUOTE_STRINGS` flags to provide finer grained control of ``None`` and empty strings by - :class:`~csv.writer` objects. + :class:`csv.writer` objects. + +dis +--- + +* Pseudo instruction opcodes (which are used by the compiler but + do not appear in executable bytecode) are now exposed in the + :mod:`dis` module. + :opcode:`HAVE_ARGUMENT` is still relevant to real opcodes, + but it is not useful for pseudo instructions. Use the new + :data:`dis.hasarg` collection instead. + (Contributed by Irit Katriel in :gh:`94216`.) + +* Add the :data:`dis.hasexc` collection to signify instructions that set + an exception handler. (Contributed by Irit Katriel in :gh:`94216`.) + +fractions +--------- + +* Objects of type :class:`fractions.Fraction` now support float-style + formatting. (Contributed by Mark Dickinson in :gh:`100161`.) + +importlib.resources +------------------- + +* :func:`importlib.resources.as_file` now supports resource directories. + (Contributed by Jason R. Coombs in :gh:`97930`.) + +* Rename first parameter of :func:`importlib.resources.files` to *anchor*. + (Contributed by Jason R. Coombs in :gh:`100598`.) inspect ------- @@ -368,68 +728,34 @@ inspect * Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals` for determining the current state of asynchronous generators. - (Contributed by Thomas Krennwallner in :issue:`35759`.) + (Contributed by Thomas Krennwallner in :gh:`79940`.) * The performance of :func:`inspect.getattr_static` has been considerably improved. Most calls to the function should be at least 2x faster than they were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex Waygood in :gh:`103193`.) -pathlib -------- +itertools +--------- -* Add support for subclassing :class:`pathlib.PurePath` and - :class:`~pathlib.Path`, plus their Posix- and Windows-specific variants. - Subclasses may override the :meth:`~pathlib.PurePath.with_segments` method - to pass information between path instances. - -* Add :meth:`~pathlib.Path.walk` for walking the directory trees and generating - all file or directory names within them, similar to :func:`os.walk`. - (Contributed by Stanislav Zmiev in :gh:`90385`.) - -* Add *walk_up* optional parameter to :meth:`pathlib.PurePath.relative_to` - to allow the insertion of ``..`` entries in the result; this behavior is - more consistent with :func:`os.path.relpath`. - (Contributed by Domenico Ragusa in :issue:`40358`.) - -* Add :meth:`pathlib.Path.is_junction` as a proxy to :func:`os.path.isjunction`. - (Contributed by Charles Machalow in :gh:`99547`.) - - -dis ---- - -* Pseudo instruction opcodes (which are used by the compiler but - do not appear in executable bytecode) are now exposed in the - :mod:`dis` module. - :opcode:`HAVE_ARGUMENT` is still relevant to real opcodes, - but it is not useful for pseudo instructions. Use the new - :data:`~dis.hasarg` collection instead. - (Contributed by Irit Katriel in :gh:`94216`.) - -fractions ---------- - -* Objects of type :class:`fractions.Fraction` now support float-style - formatting. (Contributed by Mark Dickinson in :gh:`100161`.) - -itertools ---------- - -* Added :class:`itertools.batched()` for collecting into even-sized - tuples where the last batch may be shorter than the rest. - (Contributed by Raymond Hettinger in :gh:`98363`.) +* Add :class:`itertools.batched()` for collecting into even-sized + tuples where the last batch may be shorter than the rest. + (Contributed by Raymond Hettinger in :gh:`98363`.) math ---- -* Added :func:`math.sumprod` for computing a sum of products. +* Add :func:`math.sumprod` for computing a sum of products. (Contributed by Raymond Hettinger in :gh:`100485`.) +* Extend :func:`math.nextafter` to include a *steps* argument + for moving up or down multiple steps at a time. + (By Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) + os -- -* Add :data:`os.PIDFD_NONBLOCK` to open a file descriptor +* Add :const:`os.PIDFD_NONBLOCK` to open a file descriptor for a process with :func:`os.pidfd_open` in non-blocking mode. (Contributed by Kumar Aditya in :gh:`93312`.) @@ -460,6 +786,30 @@ os.path * Add :func:`os.path.splitroot` to split a path into a triad ``(drive, root, tail)``. (Contributed by Barney Gale in :gh:`101000`.) +pathlib +------- + +* Add support for subclassing :class:`pathlib.PurePath` and + :class:`pathlib.Path`, plus their Posix- and Windows-specific variants. + Subclasses may override the :meth:`pathlib.PurePath.with_segments` method + to pass information between path instances. + +* Add :meth:`pathlib.Path.walk` for walking the directory trees and generating + all file or directory names within them, similar to :func:`os.walk`. + (Contributed by Stanislav Zmiev in :gh:`90385`.) + +* Add *walk_up* optional parameter to :meth:`pathlib.PurePath.relative_to` + to allow the insertion of ``..`` entries in the result; this behavior is + more consistent with :func:`os.path.relpath`. + (Contributed by Domenico Ragusa in :gh:`84538`.) + +* Add :meth:`pathlib.Path.is_junction` as a proxy to :func:`os.path.isjunction`. + (Contributed by Charles Machalow in :gh:`99547`.) + +* Add *case_sensitive* optional parameter to :meth:`pathlib.Path.glob`, + :meth:`pathlib.Path.rglob` and :meth:`pathlib.PurePath.match` for matching + the path's case sensitivity, allowing for more precise control over the matching process. + pdb --- @@ -468,6 +818,15 @@ pdb value. (Contributed by Tian Gao in :gh:`103693`.) +random +------ + +* Add :func:`random.binomialvariate`. + (Contributed by Raymond Hettinger in :gh:`81620`.) + +* Add a default of ``lambd=1.0`` to :func:`random.expovariate`. + (Contributed by Raymond Hettinger in :gh:`100234`.) + shutil ------ @@ -479,8 +838,7 @@ shutil * :func:`shutil.rmtree` now accepts a new argument *onexc* which is an error handler like *onerror* but which expects an exception instance - rather than a *(typ, val, tb)* triplet. *onerror* is deprecated and - will be removed in Python 3.14. + rather than a *(typ, val, tb)* triplet. *onerror* is deprecated. (Contributed by Irit Katriel in :gh:`102828`.) * :func:`shutil.which` now consults the *PATHEXT* environment variable to @@ -504,70 +862,60 @@ sqlite3 * Add a :ref:`command-line interface `. (Contributed by Erlend E. Aasland in :gh:`77617`.) -* Add the :attr:`~sqlite3.Connection.autocommit` attribute - to :class:`~sqlite3.Connection` - and the *autocommit* parameter to :func:`~sqlite3.connect` +* Add the :attr:`sqlite3.Connection.autocommit` attribute + to :class:`sqlite3.Connection` + and the *autocommit* parameter to :func:`sqlite3.connect` to control :pep:`249`-compliant :ref:`transaction handling `. (Contributed by Erlend E. Aasland in :gh:`83638`.) * Add *entrypoint* keyword-only parameter to - :meth:`~sqlite3.Connection.load_extension`, + :meth:`sqlite3.Connection.load_extension`, for overriding the SQLite extension entry point. (Contributed by Erlend E. Aasland in :gh:`103015`.) -* Add :meth:`~sqlite3.Connection.getconfig` and - :meth:`~sqlite3.Connection.setconfig` to :class:`~sqlite3.Connection` +* Add :meth:`sqlite3.Connection.getconfig` and + :meth:`sqlite3.Connection.setconfig` to :class:`sqlite3.Connection` to make configuration changes to a database connection. (Contributed by Erlend E. Aasland in :gh:`103489`.) -threading ---------- - -* Add :func:`threading.settrace_all_threads` and - :func:`threading.setprofile_all_threads` that allow to set tracing and - profiling functions in all running threads in addition to the calling one. - (Contributed by Pablo Galindo in :gh:`93503`.) - -types ------ - -* Add :func:`types.get_original_bases` to allow for further introspection of - :ref:`user-defined-generics` when subclassed. (Contributed by - James Hilton-Balfe and Alex Waygood in :gh:`101827`.) - -unicodedata ------------ - -* The Unicode database has been updated to version 15.0.0. (Contributed by - Benjamin Peterson in :gh:`96734`). - -unittest --------- +statistics +---------- -Added ``--durations`` command line option, showing the N slowest test cases:: +* Extend :func:`statistics.correlation` to include as a ``ranked`` method + for computing the Spearman correlation of ranked data. + (Contributed by Raymond Hettinger in :gh:`95861`.) - python3 -m unittest --durations=3 lib.tests.test_threading - ..... - Slowest test durations - ---------------------------------------------------------------------- - 1.210s test_timeout (Lib.test.test_threading.BarrierTests) - 1.003s test_default_timeout (Lib.test.test_threading.BarrierTests) - 0.518s test_timeout (Lib.test.test_threading.EventTests) +sys +--- - (0.000 durations hidden. Use -v to show these durations.) - ---------------------------------------------------------------------- - Ran 158 tests in 9.869s +* Add the :mod:`sys.monitoring` namespace to expose the new :ref:`PEP 669 + ` monitoring API. + (Contributed by Mark Shannon in :gh:`103082`.) - OK (skipped=3) +* Add :func:`sys.activate_stack_trampoline` and + :func:`sys.deactivate_stack_trampoline` for activating and deactivating + stack profiler trampolines, + and :func:`sys.is_stack_trampoline_active` for querying if stack profiler + trampolines are active. + (Contributed by Pablo Galindo and Christian Heimes + with contributions from Gregory P. Smith [Google] and Mark Shannon + in :gh:`96123`.) -(Contributed by Giampaolo Rodola in :issue:`4080`) +* Add :data:`sys.last_exc` which holds the last unhandled exception that + was raised (for post-mortem debugging use cases). Deprecate the + three fields that have the same information in its legacy form: + :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback`. + (Contributed by Irit Katriel in :gh:`102778`.) -uuid ----- +* :func:`sys._current_exceptions` now returns a mapping from thread-id to an + exception instance, rather than to a ``(typ, exc, tb)`` tuple. + (Contributed by Irit Katriel in :gh:`103176`.) -* Add a :ref:`command-line interface `. - (Contributed by Adam Chhina in :gh:`88597`.) +* :func:`sys.setrecursionlimit` and :func:`sys.getrecursionlimit`. + The recursion limit now applies only to Python code. Builtin functions do + not use the recursion limit, but are protected by a different mechanism + that prevents recursion from causing a virtual machine crash. tempfile -------- @@ -579,14 +927,43 @@ tempfile .. _whatsnew-typing-py312: +threading +--------- + +* Add :func:`threading.settrace_all_threads` and + :func:`threading.setprofile_all_threads` that allow to set tracing and + profiling functions in all running threads in addition to the calling one. + (Contributed by Pablo Galindo in :gh:`93503`.) + +tkinter +------- + +* ``tkinter.Canvas.coords()`` now flattens its arguments. + It now accepts not only coordinates as separate arguments + (``x1, y1, x2, y2, ...``) and a sequence of coordinates + (``[x1, y1, x2, y2, ...]``), but also coordinates grouped in pairs + (``(x1, y1), (x2, y2), ...`` and ``[(x1, y1), (x2, y2), ...]``), + like ``create_*()`` methods. + (Contributed by Serhiy Storchaka in :gh:`94473`.) + +tokenize +-------- + +* The :mod:`tokenize` module includes the changes introduced in :pep:`701`. + (Contributed by Marta Gómez Macías and Pablo Galindo in :gh:`102856`.) + See :ref:`whatsnew312-porting-to-python312` for more information on the + changes to the :mod:`tokenize` module. + +types +----- + +* Add :func:`types.get_original_bases` to allow for further introspection of + :ref:`user-defined-generics` when subclassed. (Contributed by + James Hilton-Balfe and Alex Waygood in :gh:`101827`.) + typing ------ -* Add :func:`typing.override`, an override decorator telling to static type - checkers to verify that a method overrides some method or attribute of the - same name on a base class, as per :pep:`698`. (Contributed by Steven Troxler in - :gh:`101564`.) - * :func:`isinstance` checks against :func:`runtime-checkable protocols ` now use :func:`inspect.getattr_static` rather than :func:`hasattr` to lookup whether @@ -631,39 +1008,57 @@ typing or more members may be slower than in Python 3.11. (Contributed by Alex Waygood in :gh:`74690` and :gh:`103193`.) -sys ---- +* All :data:`typing.TypedDict` and :data:`typing.NamedTuple` classes now have the + ``__orig_bases__`` attribute. (Contributed by Adrian Garcia Badaracco in + :gh:`103699`.) -* Add :func:`sys.activate_stack_trampoline` and - :func:`sys.deactivate_stack_trampoline` for activating and deactivating - stack profiler trampolines, - and :func:`sys.is_stack_trampoline_active` for querying if stack profiler - trampolines are active. - (Contributed by Pablo Galindo and Christian Heimes - with contributions from Gregory P. Smith [Google] and Mark Shannon - in :gh:`96123`.) +* Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. + (Contributed by Erik De Bonte in :gh:`99957`.) -* Add :data:`sys.last_exc` which holds the last unhandled exception that - was raised (for post-mortem debugging use cases). Deprecate the - three fields that have the same information in its legacy form: - :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback`. - (Contributed by Irit Katriel in :gh:`102778`.) +unicodedata +----------- -* :func:`sys._current_exceptions` now returns a mapping from thread-id to an - exception instance, rather than to a ``(typ, exc, tb)`` tuple. - (Contributed by Irit Katriel in :gh:`103176`.) +* The Unicode database has been updated to version 15.0.0. (Contributed by + Benjamin Peterson in :gh:`96734`). + +unittest +-------- + +Add a ``--durations`` command line option, showing the N slowest test cases:: + + python3 -m unittest --durations=3 lib.tests.test_threading + ..... + Slowest test durations + ---------------------------------------------------------------------- + 1.210s test_timeout (Lib.test.test_threading.BarrierTests) + 1.003s test_default_timeout (Lib.test.test_threading.BarrierTests) + 0.518s test_timeout (Lib.test.test_threading.EventTests) + + (0.000 durations hidden. Use -v to show these durations.) + ---------------------------------------------------------------------- + Ran 158 tests in 9.869s + + OK (skipped=3) + +(Contributed by Giampaolo Rodola in :gh:`48330`) + +uuid +---- + +* Add a :ref:`command-line interface `. + (Contributed by Adam Chhina in :gh:`88597`.) Optimizations ============= -* Removed ``wstr`` and ``wstr_length`` members from Unicode objects. +* Remove ``wstr`` and ``wstr_length`` members from Unicode objects. It reduces object size by 8 or 16 bytes on 64bit platform. (:pep:`623`) (Contributed by Inada Naoki in :gh:`92536`.) -* Added experimental support for using the BOLT binary optimizer in the build +* Add experimental support for using the BOLT binary optimizer in the build process, which improves performance by 1-5%. - (Contributed by Kevin Modzelewski in :gh:`90536` and tuned by Dong-hee Na in :gh:`101525`) + (Contributed by Kevin Modzelewski in :gh:`90536` and tuned by Donghee Na in :gh:`101525`) * Speed up the regular expression substitution (functions :func:`re.sub` and :func:`re.subn` and corresponding :class:`!re.Pattern` methods) for @@ -671,20 +1066,63 @@ Optimizations (Contributed by Serhiy Storchaka in :gh:`91524`.) * Speed up :class:`asyncio.Task` creation by deferring expensive string formatting. - (Contributed by Itamar O in :gh:`103793`.) + (Contributed by Itamar Oren in :gh:`103793`.) + +* The :func:`tokenize.tokenize` and :func:`tokenize.generate_tokens` functions are + up to 64% faster as a side effect of the changes required to cover :pep:`701` in + the :mod:`tokenize` module. (Contributed by Marta Gómez Macías and Pablo Galindo + in :gh:`102856`.) + +* Speed up :func:`super` method calls and attribute loads via the + new :opcode:`LOAD_SUPER_ATTR` instruction. (Contributed by Carl Meyer and + Vladimir Matveev in :gh:`103497`.) CPython bytecode changes ======================== -* Removed the :opcode:`LOAD_METHOD` instruction. It has been merged into +* Remove the :opcode:`!LOAD_METHOD` instruction. It has been merged into :opcode:`LOAD_ATTR`. :opcode:`LOAD_ATTR` will now behave like the old - :opcode:`LOAD_METHOD` instruction if the low bit of its oparg is set. + :opcode:`!LOAD_METHOD` instruction if the low bit of its oparg is set. (Contributed by Ken Jin in :gh:`93429`.) -* Removed the :opcode:`!JUMP_IF_FALSE_OR_POP` and :opcode:`!JUMP_IF_TRUE_OR_POP` +* Remove the :opcode:`!JUMP_IF_FALSE_OR_POP` and :opcode:`!JUMP_IF_TRUE_OR_POP` instructions. (Contributed by Irit Katriel in :gh:`102859`.) +* Remove the :opcode:`!PRECALL` instruction. (Contributed by Mark Shannon in + :gh:`92925`.) + +* Add the :opcode:`BINARY_SLICE` and :opcode:`STORE_SLICE` instructions. + (Contributed by Mark Shannon in :gh:`94163`.) + +* Add the :opcode:`CALL_INTRINSIC_1` instructions. + (Contributed by Mark Shannon in :gh:`99005`.) + +* Add the :opcode:`CALL_INTRINSIC_2` instruction. + (Contributed by Irit Katriel in :gh:`101799`.) + +* Add the :opcode:`CLEANUP_THROW` instruction. + (Contributed by Brandt Bucher in :gh:`90997`.) + +* Add the :opcode:`!END_SEND` instruction. + (Contributed by Mark Shannon in :gh:`103082`.) + +* Add the :opcode:`LOAD_FAST_AND_CLEAR` instruction as part of the + implementation of :pep:`709`. (Contributed by Carl Meyer in :gh:`101441`.) + +* Add the :opcode:`LOAD_FAST_CHECK` instruction. + (Contributed by Dennis Sweeney in :gh:`93143`.) + +* Add the :opcode:`LOAD_FROM_DICT_OR_DEREF`, :opcode:`LOAD_FROM_DICT_OR_GLOBALS`, + and :opcode:`LOAD_LOCALS` opcodes as part of the implementation of :pep:`695`. + Remove the :opcode:`!LOAD_CLASSDEREF` opcode, which can be replaced with + :opcode:`LOAD_LOCALS` plus :opcode:`LOAD_FROM_DICT_OR_DEREF`. (Contributed + by Jelle Zijlstra in :gh:`103764`.) + +* Add the :opcode:`LOAD_SUPER_ATTR` instruction. (Contributed by Carl Meyer and + Vladimir Matveev in :gh:`103497`.) + +* Add the :opcode:`RETURN_CONST` instruction. (Contributed by Wenyang Wang in :gh:`101632`.) Demos and Tools =============== @@ -703,74 +1141,194 @@ Demos and Tools Deprecated ========== -* :class:`typing.Hashable` and :class:`typing.Sized` aliases for :class:`collections.abc.Hashable` - and :class:`collections.abc.Sized`. (:gh:`94309`.) +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are deprecated + and will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`92248`.) -* The :mod:`sqlite3` :ref:`default adapters and converters - ` are now deprecated. - Instead, use the :ref:`sqlite3-adapter-converter-recipes` - and tailor them to your needs. - (Contributed by Erlend E. Aasland in :gh:`90016`.) +* :mod:`ast`: The following :mod:`ast` features have been deprecated in documentation since + Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime + when they are accessed or used, and will be removed in Python 3.14: -* In :meth:`~sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted - when :ref:`named placeholders ` are used together with - parameters supplied as a :term:`sequence` instead of as a :class:`dict`. - Starting from Python 3.14, using named placeholders with parameters supplied - as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. - (Contributed by Erlend E. Aasland in :gh:`101698`.) + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` -* The 3-arg signatures (type, value, traceback) of :meth:`~coroutine.throw`, - :meth:`~generator.throw` and :meth:`~agen.athrow` are deprecated and - may be removed in a future version of Python. Use the single-arg versions - of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.) + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) -* :exc:`DeprecationWarning` is now raised when ``__package__`` on a - module differs from ``__spec__.parent`` (previously it was - :exc:`ImportWarning`). - (Contributed by Brett Cannon in :gh:`65961`.) +* :mod:`asyncio`: -* The :meth:`~asyncio.get_event_loop` method of the - default event loop policy now emits a :exc:`DeprecationWarning` if there - is no current event loop set and it decides to create one. - (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) + * The child watcher classes :class:`asyncio.MultiLoopChildWatcher`, + :class:`asyncio.FastChildWatcher`, :class:`asyncio.AbstractChildWatcher` + and :class:`asyncio.SafeChildWatcher` are deprecated and + will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) -* The :mod:`xml.etree.ElementTree` module now emits :exc:`DeprecationWarning` - when testing the truth value of an :class:`xml.etree.ElementTree.Element`. - Before, the Python implementation emitted :exc:`FutureWarning`, and the C - implementation emitted nothing. + * :func:`asyncio.set_child_watcher`, :func:`asyncio.get_child_watcher`, + :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + and will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) -* In accordance with :pep:`699`, the ``ma_version_tag`` field in :c:type:`PyDictObject` - is deprecated for extension modules. Accessing this field will generate a compiler - warning at compile time. This field will be removed in Python 3.14. - (Contributed by Ramvikrams and Kumar Aditya in :gh:`101193`. PEP by Ken Jin.) + * The :meth:`~asyncio.get_event_loop` method of the + default event loop policy now emits a :exc:`DeprecationWarning` if there + is no current event loop set and it decides to create one. + (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) + +* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are deprecated and + replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. + (Contributed by Prince Roshan in :gh:`103636`.) + +* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. + Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`datetime`: :class:`datetime.datetime`'s :meth:`~datetime.datetime.utcnow` and + :meth:`~datetime.datetime.utcfromtimestamp` are deprecated and will be + removed in a future version. Instead, use timezone-aware objects to represent + datetimes in UTC: respectively, call :meth:`~datetime.datetime.now` and + :meth:`~datetime.datetime.fromtimestamp` with the *tz* parameter set to + :const:`datetime.UTC`. + (Contributed by Paul Ganssle in :gh:`103857`.) + +* :mod:`email`: Deprecate the *isdst* parameter in :func:`email.utils.localtime`. + (Contributed by Alan Williams in :gh:`72346`.) + +* :mod:`importlib.abc`: Deprecated the following classes, scheduled for removal in + Python 3.14: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) -* The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on - Windows are deprecated. In a future release, they will contain the last - metadata change time, consistent with other platforms. For now, they still - contain the creation time, which is also available in the new ``st_birthtime`` - field. (Contributed by Steve Dower in :gh:`99726`.) +* :mod:`itertools`: Deprecate the support for copy, deepcopy, and pickle operations, + which is undocumented, inefficient, historically buggy, and inconsistent. + This will be removed in 3.14 for a significant reduction in code + volume and maintenance burden. + (Contributed by Raymond Hettinger in :gh:`101588`.) -* The :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback` +* :mod:`multiprocessing`: In Python 3.14, the default :mod:`multiprocessing` + start method will change to a safer one on Linux, BSDs, + and other non-macOS POSIX platforms where ``'fork'`` is currently + the default (:gh:`84559`). Adding a runtime warning about this was deemed too + disruptive as the majority of code is not expected to care. Use the + :func:`~multiprocessing.get_context` or + :func:`~multiprocessing.set_start_method` APIs to explicitly specify when + your code *requires* ``'fork'``. See :ref:`contexts and start methods + `. + +* :mod:`pkgutil`: :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` + are deprecated and will be removed in Python 3.14; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + +* :mod:`pty`: The module has two undocumented ``master_open()`` and ``slave_open()`` + functions that have been deprecated since Python 2 but only gained a + proper :exc:`DeprecationWarning` in 3.12. Remove them in 3.14. + (Contributed by Soumendra Ganguly and Gregory P. Smith in :gh:`85984`.) + +* :mod:`os`: + + * The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on + Windows are deprecated. In a future release, they will contain the last + metadata change time, consistent with other platforms. For now, they still + contain the creation time, which is also available in the new ``st_birthtime`` + field. (Contributed by Steve Dower in :gh:`99726`.) + + * On POSIX platforms, :func:`os.fork` can now raise a + :exc:`DeprecationWarning` when it can detect being called from a + multithreaded process. There has always been a fundamental incompatibility + with the POSIX platform when doing so. Even if such code *appeared* to work. + We added the warning to to raise awareness as issues encounted by code doing + this are becoming more frequent. See the :func:`os.fork` documentation for + more details along with `this discussion on fork being incompatible with threads + `_ for *why* we're now surfacing this + longstanding platform compatibility problem to developers. + + When this warning appears due to usage of :mod:`multiprocessing` or + :mod:`concurrent.futures` the fix is to use a different + :mod:`multiprocessing` start method such as ``"spawn"`` or ``"forkserver"``. + +* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated; + use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) + +* :mod:`sqlite3`: + + * :ref:`default adapters and converters + ` are now deprecated. + Instead, use the :ref:`sqlite3-adapter-converter-recipes` + and tailor them to your needs. + (Contributed by Erlend E. Aasland in :gh:`90016`.) + + * In :meth:`~sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted + when :ref:`named placeholders ` are used together with + parameters supplied as a :term:`sequence` instead of as a :class:`dict`. + Starting from Python 3.14, using named placeholders with parameters supplied + as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`101698`.) + +* :mod:`sys`: The :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback` fields are deprecated. Use :data:`sys.last_exc` instead. (Contributed by Irit Katriel in :gh:`102778`.) -* The *onerror* argument of :func:`shutil.rmtree` is deprecated as will be removed - in Python 3.14. Use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) - -* Extracting tar archives without specifying *filter* is deprecated until +* :mod:`tarfile`: Extracting tar archives without specifying *filter* is deprecated until Python 3.14, when ``'data'`` filter will become the default. See :ref:`tarfile-extraction-filter` for details. -* ``calendar.January`` and ``calendar.February`` constants are deprecated and - replaced by :data:`calendar.Month.JANUARY` and :data:`calendar.Month.FEBRUARY`. - (Contributed by Prince Roshan in :gh:`103636`.) +* :mod:`typing`: + + * :class:`typing.Hashable` and :class:`typing.Sized`, aliases for + :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are + deprecated. (:gh:`94309`.) + + * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a + :exc:`DeprecationWarning` to be emitted when it is used. + (Contributed by Alex Waygood in :gh:`91896`.) + +* :mod:`xml.etree.ElementTree`: The module now emits :exc:`DeprecationWarning` + when testing the truth value of an :class:`xml.etree.ElementTree.Element`. + Before, the Python implementation emitted :exc:`FutureWarning`, and the C + implementation emitted nothing. + (Contributed by Jacob Walls in :gh:`83122`.) + +* The 3-arg signatures (type, value, traceback) of :meth:`coroutine throw() + `, :meth:`generator throw() ` and + :meth:`async generator throw() ` are deprecated and + may be removed in a future version of Python. Use the single-arg versions + of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.) + +* :exc:`DeprecationWarning` is now raised when ``__package__`` on a + module differs from ``__spec__.parent`` (previously it was + :exc:`ImportWarning`). + (Contributed by Brett Cannon in :gh:`65961`.) + +* Setting ``__package__`` or ``__cached__`` on a module is deprecated, + and will cease to be set or taken into consideration by the import system in Python 3.14. + (Contributed by Brett Cannon in :gh:`65961`.) * The bitwise inversion operator (``~``) on bool is deprecated. It will throw an error in Python 3.14. Use ``not`` for logical negation of bools instead. In the rare case that you really need the bitwise inversion of the underlying - ``int``, convert to int explicitly with ``~int(x)``. (Contributed by Tim Hoffmann + ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) +* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in + Python 3.10 via :pep:`626`, + but it only got a proper :exc:`DeprecationWarning` in 3.12, + therefore it will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`101866`.) + Pending Removal in Python 3.13 ------------------------------ @@ -779,111 +1337,112 @@ and will be removed in Python 3.13. Modules (see :pep:`594`): -* :mod:`aifc` -* :mod:`audioop` -* :mod:`cgi` -* :mod:`cgitb` -* :mod:`chunk` -* :mod:`crypt` -* :mod:`imghdr` -* :mod:`mailcap` -* :mod:`msilib` -* :mod:`nis` -* :mod:`nntplib` -* :mod:`ossaudiodev` -* :mod:`pipes` -* :mod:`sndhdr` -* :mod:`spwd` -* :mod:`sunau` -* :mod:`telnetlib` -* :mod:`uu` -* :mod:`xdrlib` +* :mod:`!aifc` +* :mod:`!audioop` +* :mod:`!cgi` +* :mod:`!cgitb` +* :mod:`!chunk` +* :mod:`!crypt` +* :mod:`!imghdr` +* :mod:`!mailcap` +* :mod:`!msilib` +* :mod:`!nis` +* :mod:`!nntplib` +* :mod:`!ossaudiodev` +* :mod:`!pipes` +* :mod:`!sndhdr` +* :mod:`!spwd` +* :mod:`!sunau` +* :mod:`!telnetlib` +* :mod:`!uu` +* :mod:`!xdrlib` + +Other modules: + +* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) APIs: * :class:`!configparser.LegacyInterpolation` (:gh:`90765`) -* :func:`locale.getdefaultlocale` (:gh:`90817`) +* ``locale.resetlocale()`` (:gh:`90817`) * :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) * :func:`!unittest.findTestCases` (:gh:`50096`) -* :func:`!unittest.makeSuite` (:gh:`50096`) * :func:`!unittest.getTestCaseNames` (:gh:`50096`) +* :func:`!unittest.makeSuite` (:gh:`50096`) +* :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) +* :class:`classmethod` descriptor chaining (:gh:`89519`) Pending Removal in Python 3.14 ------------------------------ -* Deprecated the following :mod:`importlib.abc` classes, scheduled for removal in - Python 3.14: +The following APIs have been deprecated +and will be removed in Python 3.14. + +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` + +* :mod:`ast`: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + +* :mod:`asyncio`: + + * :class:`!asyncio.MultiLoopChildWatcher` + * :class:`!asyncio.FastChildWatcher` + * :class:`!asyncio.AbstractChildWatcher` + * :class:`!asyncio.SafeChildWatcher` + * :func:`!asyncio.set_child_watcher` + * :func:`!asyncio.get_child_watcher`, + * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` + +* :mod:`collections.abc`: :class:`!collections.abc.ByteString`. + +* :mod:`email`: the *isdst* parameter in :func:`email.utils.localtime`. + +* :mod:`importlib.abc`: * :class:`!importlib.abc.ResourceReader` * :class:`!importlib.abc.Traversable` * :class:`!importlib.abc.TraversableResources` - Use :mod:`importlib.resources.abc` classes instead: +* :mod:`itertools`: Support for copy, deepcopy, and pickle operations. - * :class:`importlib.resources.abc.Traversable` - * :class:`importlib.resources.abc.TraversableResources` +* :mod:`pkgutil`: - (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + * :func:`!pkgutil.find_loader` + * :func:`!pkgutil.get_loader`. -* Deprecated :class:`collections.abc.ByteString`. - Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) +* :mod:`pty`: -* :class:`typing.ByteString`, deprecated since Python 3.9, now causes a - :exc:`DeprecationWarning` to be emitted when it is used. + * :func:`!pty.master_open` + * :func:`!pty.slave_open` -* Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable - bases using the C API. +* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` -* Deprecated the *isdst* parameter in :func:`email.utils.localtime`. - (Contributed by Alan Williams in :gh:`72346`.) +* :mod:`typing`: :class:`!typing.ByteString` -* ``__package__`` and ``__cached__`` will cease to be set or taken - into consideration by the import system (:gh:`97879`). +* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element`. -* Testing the truth value of an :class:`xml.etree.ElementTree.Element` - is deprecated and will raise an exception in Python 3.14. +* The ``__package__`` and ``__cached__`` attributes on module objects. -* The default :mod:`multiprocessing` start method will change to a safer one on - Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently - the default (:gh:`84559`). Adding a runtime warning about this was deemed too - disruptive as the majority of code is not expected to care. Use the - :func:`~multiprocessing.get_context` or - :func:`~multiprocessing.set_start_method` APIs to explicitly specify when - your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. +* The :attr:`~codeobject.co_lnotab` attribute of code objects. -* :mod:`pty` has two undocumented ``master_open()`` and ``slave_open()`` - functions that have been deprecated since Python 2 but only gained a - proper :exc:`DeprecationWarning` in 3.12. Remove them in 3.14. - -* Accessing ``co_lnotab`` was deprecated in :pep:`626` since 3.10 - and was planned to be removed in 3.12 - but it only got a proper :exc:`DeprecationWarning` in 3.12. - May be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`101866`.) +Pending Removal in Python 3.15 +------------------------------ -* The *onerror* argument of :func:`shutil.rmtree` is deprecated in 3.12, - and will be removed in 3.14. +The following APIs have been deprecated +and will be removed in Python 3.15. -* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` - now raise :exc:`DeprecationWarning`; - use :func:`importlib.util.find_spec` instead. - (Contributed by Nikita Sobolev in :gh:`97850`.) - -* The following :mod:`ast` features have been deprecated in documentation since - Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime - when they are accessed or used, and will be removed in Python 3.14: +APIs: - * :class:`!ast.Num` - * :class:`!ast.Str` - * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` - * :class:`!ast.Ellipsis` +* :func:`locale.getdefaultlocale` (:gh:`90817`) - Use :class:`ast.Constant` instead. - (Contributed by Serhiy Storchaka in :gh:`90953`.) Pending Removal in Future Versions ---------------------------------- @@ -891,6 +1450,8 @@ Pending Removal in Future Versions The following APIs were deprecated in earlier Python versions and will be removed, although there is currently no date scheduled for their removal. +* :mod:`array`'s ``'u'`` format code (:gh:`57281`) + * :class:`typing.Text` (:gh:`92332`) * Currently Python accepts numeric literals immediately followed by keywords, @@ -906,12 +1467,40 @@ although there is currently no date scheduled for their removal. Removed ======= -* Remove the ``distutils`` package. It was deprecated in Python 3.10 by +asynchat and asyncore +--------------------- + +* These two modules have been removed + according to the schedule in :pep:`594`, + having been deprecated in Python 3.6. + Use :mod:`asyncio` instead. + (Contributed by Nikita Sobolev in :gh:`96580`.) + +configparser +------------ + +* Several names deprecated in the :mod:`configparser` way back in 3.2 have + been removed per :gh:`89336`: + + * :class:`configparser.ParsingError` no longer has a ``filename`` attribute + or argument. Use the ``source`` attribute and argument instead. + * :mod:`configparser` no longer has a ``SafeConfigParser`` class. Use the + shorter :class:`~configparser.ConfigParser` name instead. + * :class:`configparser.ConfigParser` no longer has a ``readfp`` method. + Use :meth:`~configparser.ConfigParser.read_file` instead. + +distutils +--------- + +* Remove the :py:mod:`!distutils` package. It was deprecated in Python 3.10 by :pep:`632` "Deprecate distutils module". For projects still using ``distutils`` and cannot be updated to something else, the ``setuptools`` project can be installed: it still provides ``distutils``. (Contributed by Victor Stinner in :gh:`92584`.) +ensurepip +--------- + * Remove the bundled setuptools wheel from :mod:`ensurepip`, and stop installing setuptools in environments created by :mod:`venv`. @@ -930,54 +1519,140 @@ Removed (Contributed by Pradyun Gedam in :gh:`95299`.) -* Removed many old deprecated :mod:`unittest` features: +enum +---- - - A number of :class:`~unittest.TestCase` method aliases: +* Remove :mod:`enum`'s ``EnumMeta.__getattr__``, which is no longer needed for + enum attribute access. + (Contributed by Ethan Furman in :gh:`95083`.) - ============================ =============================== =============== - Deprecated alias Method Name Deprecated in - ============================ =============================== =============== - ``failUnless`` :meth:`.assertTrue` 3.1 - ``failIf`` :meth:`.assertFalse` 3.1 - ``failUnlessEqual`` :meth:`.assertEqual` 3.1 - ``failIfEqual`` :meth:`.assertNotEqual` 3.1 - ``failUnlessAlmostEqual`` :meth:`.assertAlmostEqual` 3.1 - ``failIfAlmostEqual`` :meth:`.assertNotAlmostEqual` 3.1 - ``failUnlessRaises`` :meth:`.assertRaises` 3.1 - ``assert_`` :meth:`.assertTrue` 3.2 - ``assertEquals`` :meth:`.assertEqual` 3.2 - ``assertNotEquals`` :meth:`.assertNotEqual` 3.2 - ``assertAlmostEquals`` :meth:`.assertAlmostEqual` 3.2 - ``assertNotAlmostEquals`` :meth:`.assertNotAlmostEqual` 3.2 - ``assertRegexpMatches`` :meth:`.assertRegex` 3.2 - ``assertRaisesRegexp`` :meth:`.assertRaisesRegex` 3.2 - ``assertNotRegexpMatches`` :meth:`.assertNotRegex` 3.5 - ============================ =============================== =============== +ftplib +------ - You can use https://github.com/isidentical/teyit to automatically modernise - your unit tests. +* Remove :mod:`ftplib`'s ``FTP_TLS.ssl_version`` class attribute: use the + *context* parameter instead. + (Contributed by Victor Stinner in :gh:`94172`.) - - Undocumented and broken :class:`~unittest.TestCase` method - ``assertDictContainsSubset`` (deprecated in Python 3.2). +gzip +---- - - Undocumented :meth:`TestLoader.loadTestsFromModule - ` parameter *use_load_tests* - (deprecated and ignored since Python 3.2). +* Remove the ``filename`` attribute of :mod:`gzip`'s :class:`gzip.GzipFile`, + deprecated since Python 2.6, use the :attr:`~gzip.GzipFile.name` attribute + instead. In write mode, the ``filename`` attribute added ``'.gz'`` file + extension if it was not present. + (Contributed by Victor Stinner in :gh:`94196`.) - - An alias of the :class:`~unittest.TextTestResult` class: - ``_TextTestResult`` (deprecated in Python 3.2). +hashlib +------- - (Contributed by Serhiy Storchaka in :issue:`45162`.) +* Remove the pure Python implementation of :mod:`hashlib`'s + :func:`hashlib.pbkdf2_hmac()`, deprecated in Python 3.10. Python 3.10 and + newer requires OpenSSL 1.1.1 (:pep:`644`): this OpenSSL version provides + a C implementation of :func:`~hashlib.pbkdf2_hmac()` which is faster. + (Contributed by Victor Stinner in :gh:`94199`.) -* Several names deprecated in the :mod:`configparser` way back in 3.2 have - been removed per :gh:`89336`: +importlib +--------- - * :class:`configparser.ParsingError` no longer has a ``filename`` attribute - or argument. Use the ``source`` attribute and argument instead. - * :mod:`configparser` no longer has a ``SafeConfigParser`` class. Use the - shorter :class:`~configparser.ConfigParser` name instead. - * :class:`configparser.ConfigParser` no longer has a ``readfp`` method. - Use :meth:`~configparser.ConfigParser.read_file` instead. +* Many previously deprecated cleanups in :mod:`importlib` have now been + completed: + + * References to, and support for :meth:`!module_repr()` has been removed. + (Contributed by Barry Warsaw in :gh:`97850`.) + + * ``importlib.util.set_package``, ``importlib.util.set_loader`` and + ``importlib.util.module_for_loader`` have all been removed. (Contributed by + Brett Cannon and Nikita Sobolev in :gh:`65961` and :gh:`97850`.) + + * Support for ``find_loader()`` and ``find_module()`` APIs have been + removed. (Contributed by Barry Warsaw in :gh:`98040`.) + + * ``importlib.abc.Finder``, ``pkgutil.ImpImporter``, and ``pkgutil.ImpLoader`` + have been removed. (Contributed by Barry Warsaw in :gh:`98040`.) + +imp +--- + +* The :mod:`!imp` module has been removed. (Contributed by Barry Warsaw in + :gh:`98040`.) + + To migrate, consult the following correspondence table: + + ================================= ======================================= + imp importlib + ================================= ======================================= + ``imp.NullImporter`` Insert ``None`` into ``sys.path_importer_cache`` + ``imp.cache_from_source()`` :func:`importlib.util.cache_from_source` + ``imp.find_module()`` :func:`importlib.util.find_spec` + ``imp.get_magic()`` :attr:`importlib.util.MAGIC_NUMBER` + ``imp.get_suffixes()`` :attr:`importlib.machinery.SOURCE_SUFFIXES`, :attr:`importlib.machinery.EXTENSION_SUFFIXES`, and :attr:`importlib.machinery.BYTECODE_SUFFIXES` + ``imp.get_tag()`` :attr:`sys.implementation.cache_tag ` + ``imp.load_module()`` :func:`importlib.import_module` + ``imp.new_module(name)`` ``types.ModuleType(name)`` + ``imp.reload()`` :func:`importlib.reload` + ``imp.source_from_cache()`` :func:`importlib.util.source_from_cache` + ``imp.load_source()`` *See below* + ================================= ======================================= + + Replace ``imp.load_source()`` with:: + + import importlib.util + import importlib.machinery + + def load_source(modname, filename): + loader = importlib.machinery.SourceFileLoader(modname, filename) + spec = importlib.util.spec_from_file_location(modname, filename, loader=loader) + module = importlib.util.module_from_spec(spec) + # The module is always executed and not cached in sys.modules. + # Uncomment the following line to cache the module. + # sys.modules[module.__name__] = module + loader.exec_module(module) + return module + +* Remove :mod:`!imp` functions and attributes with no replacements: + + * Undocumented functions: + + * ``imp.init_builtin()`` + * ``imp.load_compiled()`` + * ``imp.load_dynamic()`` + * ``imp.load_package()`` + + * ``imp.lock_held()``, ``imp.acquire_lock()``, ``imp.release_lock()``: + the locking scheme has changed in Python 3.3 to per-module locks. + * ``imp.find_module()`` constants: ``SEARCH_ERROR``, ``PY_SOURCE``, + ``PY_COMPILED``, ``C_EXTENSION``, ``PY_RESOURCE``, ``PKG_DIRECTORY``, + ``C_BUILTIN``, ``PY_FROZEN``, ``PY_CODERESOURCE``, ``IMP_HOOK``. + +io +-- + +* Remove :mod:`io`'s ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python + 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) + function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is + also a static method. + (Contributed by Victor Stinner in :gh:`94169`.) + +locale +------ + +* Remove :mod:`locale`'s :func:`!locale.format` function, deprecated in Python 3.7: + use :func:`locale.format_string` instead. + (Contributed by Victor Stinner in :gh:`94226`.) + +smtpd +----- + +* The ``smtpd`` module has been removed according to the schedule in :pep:`594`, + having been deprecated in Python 3.4.7 and 3.5.4. + Use aiosmtpd_ PyPI module or any other + :mod:`asyncio`-based server instead. + (Contributed by Oleg Iarygin in :gh:`93243`.) + +.. _aiosmtpd: https://pypi.org/project/aiosmtpd/ + +sqlite3 +------- * The following undocumented :mod:`sqlite3` features, deprecated in Python 3.10, are now removed: @@ -995,63 +1670,19 @@ Removed (Contributed by Erlend E. Aasland in :gh:`92548`.) -* ``smtpd`` has been removed according to the schedule in :pep:`594`, - having been deprecated in Python 3.4.7 and 3.5.4. - Use aiosmtpd_ PyPI module or any other - :mod:`asyncio`-based server instead. - (Contributed by Oleg Iarygin in :gh:`93243`.) - -.. _aiosmtpd: https://pypi.org/project/aiosmtpd/ - -* ``asynchat`` and ``asyncore`` have been removed - according to the schedule in :pep:`594`, - having been deprecated in Python 3.6. - Use :mod:`asyncio` instead. - (Contributed by Nikita Sobolev in :gh:`96580`.) - -* Remove ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python - 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) - function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is - also a static method. - (Contributed by Victor Stinner in :gh:`94169`.) +ssl +--- -* Remove the :func:`!ssl.RAND_pseudo_bytes` function, deprecated in Python 3.6: +* Remove :mod:`ssl`'s :func:`!ssl.RAND_pseudo_bytes` function, deprecated in Python 3.6: use :func:`os.urandom` or :func:`ssl.RAND_bytes` instead. (Contributed by Victor Stinner in :gh:`94199`.) -* :mod:`gzip`: Remove the ``filename`` attribute of :class:`gzip.GzipFile`, - deprecated since Python 2.6, use the :attr:`~gzip.GzipFile.name` attribute - instead. In write mode, the ``filename`` attribute added ``'.gz'`` file - extension if it was not present. - (Contributed by Victor Stinner in :gh:`94196`.) - * Remove the :func:`!ssl.match_hostname` function. It was deprecated in Python 3.7. OpenSSL performs hostname matching since Python 3.7, Python no longer uses the :func:`!ssl.match_hostname` function. (Contributed by Victor Stinner in :gh:`94199`.) -* Remove the :func:`!locale.format` function, deprecated in Python 3.7: - use :func:`locale.format_string` instead. - (Contributed by Victor Stinner in :gh:`94226`.) - -* :mod:`hashlib`: Remove the pure Python implementation of - :func:`hashlib.pbkdf2_hmac()`, deprecated in Python 3.10. Python 3.10 and - newer requires OpenSSL 1.1.1 (:pep:`644`): this OpenSSL version provides - a C implementation of :func:`~hashlib.pbkdf2_hmac()` which is faster. - (Contributed by Victor Stinner in :gh:`94199`.) - -* :mod:`xml.etree.ElementTree`: Remove the ``ElementTree.Element.copy()`` method of the - pure Python implementation, deprecated in Python 3.10, use the - :func:`copy.copy` function instead. The C implementation of :mod:`xml.etree.ElementTree` - has no ``copy()`` method, only a ``__copy__()`` method. - (Contributed by Victor Stinner in :gh:`94383`.) - -* :mod:`zipimport`: Remove ``find_loader()`` and ``find_module()`` methods, - deprecated in Python 3.10: use the ``find_spec()`` method instead. See - :pep:`451` for the rationale. - (Contributed by Victor Stinner in :gh:`94379`.) - * Remove the :func:`!ssl.wrap_socket` function, deprecated in Python 3.7: instead, create a :class:`ssl.SSLContext` object and call its :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses @@ -1061,26 +1692,79 @@ Removed Validation. (Contributed by Victor Stinner in :gh:`94199`.) -* Many previously deprecated cleanups in :mod:`importlib` have now been - completed: +unittest +-------- - * References to, and support for :meth:`!module_repr()` has been removed. - (Contributed by Barry Warsaw in :gh:`97850`.) +* Remove many long-deprecated :mod:`unittest` features: - * ``importlib.util.set_package`` has been removed. (Contributed by Brett - Cannon in :gh:`65961`.) + .. _unittest-TestCase-removed-aliases: - * Support for ``find_loader()`` and ``find_module()`` APIs have been - removed. (Contributed by Barry Warsaw in :gh:`98040`.) + * A number of :class:`~unittest.TestCase` method aliases: + + ============================ =============================== =============== + Deprecated alias Method Name Deprecated in + ============================ =============================== =============== + ``failUnless`` :meth:`.assertTrue` 3.1 + ``failIf`` :meth:`.assertFalse` 3.1 + ``failUnlessEqual`` :meth:`.assertEqual` 3.1 + ``failIfEqual`` :meth:`.assertNotEqual` 3.1 + ``failUnlessAlmostEqual`` :meth:`.assertAlmostEqual` 3.1 + ``failIfAlmostEqual`` :meth:`.assertNotAlmostEqual` 3.1 + ``failUnlessRaises`` :meth:`.assertRaises` 3.1 + ``assert_`` :meth:`.assertTrue` 3.2 + ``assertEquals`` :meth:`.assertEqual` 3.2 + ``assertNotEquals`` :meth:`.assertNotEqual` 3.2 + ``assertAlmostEquals`` :meth:`.assertAlmostEqual` 3.2 + ``assertNotAlmostEquals`` :meth:`.assertNotAlmostEqual` 3.2 + ``assertRegexpMatches`` :meth:`.assertRegex` 3.2 + ``assertRaisesRegexp`` :meth:`.assertRaisesRegex` 3.2 + ``assertNotRegexpMatches`` :meth:`.assertNotRegex` 3.5 + ============================ =============================== =============== + + You can use https://github.com/isidentical/teyit to automatically modernise + your unit tests. + + * Undocumented and broken :class:`~unittest.TestCase` method + ``assertDictContainsSubset`` (deprecated in Python 3.2). + + * Undocumented :meth:`TestLoader.loadTestsFromModule + ` parameter *use_load_tests* + (deprecated and ignored since Python 3.2). + + * An alias of the :class:`~unittest.TextTestResult` class: + ``_TextTestResult`` (deprecated in Python 3.2). + + (Contributed by Serhiy Storchaka in :gh:`89325`.) + +webbrowser +---------- + +* Remove support for obsolete browsers from :mod:`webbrowser`. + The removed browsers include: Grail, Mosaic, Netscape, Galeon, Skipstone, + Iceape, Firebird, and Firefox versions 35 and below (:gh:`102871`). + +xml.etree.ElementTree +--------------------- - * ``importlib.abc.Finder``, ``pkg.ImpImporter``, and ``pkg.ImpLoader`` have - been removed. (Contributed by Barry Warsaw in :gh:`98040`.) +* Remove the ``ElementTree.Element.copy()`` method of the + pure Python implementation, deprecated in Python 3.10, use the + :func:`copy.copy` function instead. The C implementation of :mod:`xml.etree.ElementTree` + has no ``copy()`` method, only a ``__copy__()`` method. + (Contributed by Victor Stinner in :gh:`94383`.) + +zipimport +--------- + +* Remove :mod:`zipimport`'s ``find_loader()`` and ``find_module()`` methods, + deprecated in Python 3.10: use the ``find_spec()`` method instead. See + :pep:`451` for the rationale. + (Contributed by Victor Stinner in :gh:`94379`.) - * The :mod:`!imp` module has been removed. (Contributed by Barry Warsaw in - :gh:`98040`.) +Others +------ -* Removed the ``suspicious`` rule from the documentation Makefile, and - removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint +* Remove the ``suspicious`` rule from the documentation :file:`Makefile` and + :file:`Doc/tools/rstlint.py`, both in favor of `sphinx-lint `_. (Contributed by Julien Palard in :gh:`98179`.) @@ -1092,15 +1776,16 @@ Removed (*ssl_context* in :mod:`imaplib`) instead. (Contributed by Victor Stinner in :gh:`94172`.) -* :mod:`ftplib`: Remove the ``FTP_TLS.ssl_version`` class attribute: use the - *context* parameter instead. - (Contributed by Victor Stinner in :gh:`94172`.) +* Remove ``Jython`` compatibility hacks from several stdlib modules and tests. + (Contributed by Nikita Sobolev in :gh:`99482`.) -* Remove support for obsolete browsers from :mod:`webbrowser`. - Removed browsers include: Grail, Mosaic, Netscape, Galeon, Skipstone, - Iceape, Firebird, and Firefox versions 35 and below (:gh:`102871`). +* Remove ``_use_broken_old_ctypes_structure_semantics_`` flag + from :mod:`ctypes` module. + (Contributed by Nikita Sobolev in :gh:`99285`.) +.. _whatsnew312-porting-to-python312: + Porting to Python 3.12 ====================== @@ -1117,9 +1802,9 @@ Changes in the Python API contain ASCII letters and digits and underscore. (Contributed by Serhiy Storchaka in :gh:`91760`.) -* Removed ``randrange()`` functionality deprecated since Python 3.10. Formerly, +* Remove ``randrange()`` functionality deprecated since Python 3.10. Formerly, ``randrange(10.0)`` losslessly converted to ``randrange(10)``. Now, it raises a - :exc:`TypeError`. Also, the exception raised for non-integral values such as + :exc:`TypeError`. Also, the exception raised for non-integer values such as ``randrange(10.5)`` or ``randrange('10')`` has been changed from :exc:`ValueError` to :exc:`TypeError`. This also prevents bugs where ``randrange(1e25)`` would silently select from a larger range than ``randrange(10**25)``. @@ -1131,7 +1816,7 @@ Changes in the Python API to :term:`filesystem encoding and error handler`. Argument files should be encoded in UTF-8 instead of ANSI Codepage on Windows. -* Removed the ``asyncore``-based ``smtpd`` module deprecated in Python 3.4.7 +* Remove the ``asyncore``-based ``smtpd`` module deprecated in Python 3.4.7 and 3.5.4. A recommended replacement is the :mod:`asyncio`-based aiosmtpd_ PyPI module. @@ -1152,7 +1837,7 @@ Changes in the Python API so only a very small set of users might be affected. This change helps with interpreter isolation. Furthermore, :mod:`syslog` is a wrapper around process-global resources, which are best managed from the main interpreter. - (Contributed by Dong-hee Na in :gh:`99127`.) + (Contributed by Donghee Na in :gh:`99127`.) * The undocumented locking behavior of :func:`~functools.cached_property` is removed, because it locked across all instances of the class, leading to high @@ -1172,12 +1857,59 @@ Changes in the Python API that may be surprising or dangerous. See :ref:`tarfile-extraction-filter` for details. +* The output of the :func:`tokenize.tokenize` and :func:`tokenize.generate_tokens` + functions is now changed due to the changes introduced in :pep:`701`. This + means that ``STRING`` tokens are not emitted any more for f-strings and the + tokens described in :pep:`701` are now produced instead: ``FSTRING_START``, + ``FSTRING_MIDDLE`` and ``FSTRING_END`` are now emitted for f-string "string" + parts in addition to the appropriate tokens for the tokenization in the + expression components. For example for the f-string ``f"start {1+1} end"`` + the old version of the tokenizer emitted:: + + 1,0-1,18: STRING 'f"start {1+1} end"' + + while the new version emits:: + + 1,0-1,2: FSTRING_START 'f"' + 1,2-1,8: FSTRING_MIDDLE 'start ' + 1,8-1,9: OP '{' + 1,9-1,10: NUMBER '1' + 1,10-1,11: OP '+' + 1,11-1,12: NUMBER '1' + 1,12-1,13: OP '}' + 1,13-1,17: FSTRING_MIDDLE ' end' + 1,17-1,18: FSTRING_END '"' + + Additionally, there may be some minor behavioral changes as a consequence of the + changes required to support :pep:`701`. Some of these changes include: + + * The ``type`` attribute of the tokens emitted when tokenizing some invalid Python + characters such as ``!`` has changed from ``ERRORTOKEN`` to ``OP``. + + * Incomplete single-line strings now also raise :exc:`tokenize.TokenError` as incomplete + multiline strings do. + + * Some incomplete or invalid Python code now raises :exc:`tokenize.TokenError` instead of + returning arbitrary ``ERRORTOKEN`` tokens when tokenizing it. + + * Mixing tabs and spaces as indentation in the same file is not supported anymore and will + raise a :exc:`TabError`. + +* The :mod:`threading` module now expects the :mod:`!_thread` module to have + an ``_is_main_interpreter`` attribute. It is a function with no + arguments that returns ``True`` if the current interpreter is the + main interpreter. + + Any library or application that provides a custom ``_thread`` module + should provide ``_is_main_interpreter()``. + (See :gh:`112826`.) + Build Changes ============= -* Python no longer uses ``setup.py`` to build shared C extension modules. +* Python no longer uses :file:`setup.py` to build shared C extension modules. Build parameters like headers and libraries are detected in ``configure`` - script. Extensions are built by ``Makefile``. Most extensions use + script. Extensions are built by :file:`Makefile`. Most extensions use ``pkg-config`` and fall back to manual detection. (Contributed by Christian Heimes in :gh:`93939`.) @@ -1188,9 +1920,9 @@ Build Changes * CPython now uses the ThinLTO option as the default link time optimization policy if the Clang compiler accepts the flag. - (Contributed by Dong-hee Na in :gh:`89536`.) + (Contributed by Donghee Na in :gh:`89536`.) -* Add ``COMPILEALL_OPTS`` variable in Makefile to override :mod:`compileall` +* Add ``COMPILEALL_OPTS`` variable in :file:`Makefile` to override :mod:`compileall` options (default: ``-j0``) in ``make install``. Also merged the 3 ``compileall`` commands into a single command to build .pyc files for all optimization levels (0, 1, 2) at once. @@ -1206,6 +1938,10 @@ Build Changes * ``PYTHON_FOR_REGEN`` now require Python 3.10 or newer. +* Autoconf 2.71 and aclocal 1.16.4 is now required to regenerate + :file:`!configure`. + (Contributed by Christian Heimes in :gh:`89886`.) + C API Changes ============= @@ -1213,8 +1949,9 @@ C API Changes New Features ------------ +.. _whatsnew312-pep697: -* :pep:`697`: Introduced the :ref:`Unstable C API tier `, +* :pep:`697`: Introduce the :ref:`Unstable C API tier `, intended for low-level tools like debuggers and JIT compilers. This API may change in each minor release of CPython without deprecation warnings. @@ -1236,14 +1973,14 @@ New Features (Contributed by Petr Viktorin in :gh:`101101`.) -* :pep:`697`: Added API for extending types whose instance memory layout is +* :pep:`697`: Add an API for extending types whose instance memory layout is opaque: - :c:member:`PyType_Spec.basicsize` can be zero or negative to specify inheriting or extending the base class size. - :c:func:`PyObject_GetTypeData` and :c:func:`PyType_GetTypeDataSize` added to allow access to subclass-specific instance data. - - :const:`Py_TPFLAGS_ITEMS_AT_END` and :c:func:`PyObject_GetItemData` + - :c:macro:`Py_TPFLAGS_ITEMS_AT_END` and :c:func:`PyObject_GetItemData` added to allow safely extending certain variable-sized types, including :c:var:`PyType_Type`. - :c:macro:`Py_RELATIVE_OFFSET` added to allow defining @@ -1251,7 +1988,7 @@ New Features (Contributed by Petr Viktorin in :gh:`103509`.) -* Added the new limited C API function :c:func:`PyType_FromMetaclass`, +* Add the new :ref:`limited C API ` function :c:func:`PyType_FromMetaclass`, which generalizes the existing :c:func:`PyType_FromModuleAndSpec` using an additional metaclass argument. (Contributed by Wenzel Jakob in :gh:`93012`.) @@ -1260,20 +1997,20 @@ New Features :ref:`the vectorcall protocol ` was added to the :ref:`Limited API `: - * :const:`Py_TPFLAGS_HAVE_VECTORCALL` + * :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` * :c:func:`PyVectorcall_NARGS` * :c:func:`PyVectorcall_Call` * :c:type:`vectorcallfunc` - The :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag is now removed from a class + The :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag is now removed from a class when the class's :py:meth:`~object.__call__` method is reassigned. This makes vectorcall safe to use with mutable types (i.e. heap types - without the immutable flag, :const:`Py_TPFLAGS_IMMUTABLETYPE`). + without the immutable flag, :c:macro:`Py_TPFLAGS_IMMUTABLETYPE`). Mutable types that do not override :c:member:`~PyTypeObject.tp_call` now inherit the ``Py_TPFLAGS_HAVE_VECTORCALL`` flag. (Contributed by Petr Viktorin in :gh:`93274`.) - The :const:`Py_TPFLAGS_MANAGED_DICT` and :const:`Py_TPFLAGS_MANAGED_WEAKREF` + The :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` flags have been added. This allows extensions classes to support object ``__dict__`` and weakrefs with less bookkeeping, using less memory and with faster access. @@ -1284,19 +2021,19 @@ New Features * :c:func:`PyObject_Vectorcall` * :c:func:`PyObject_VectorcallMethod` - * :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` + * :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET` This means that both the incoming and outgoing ends of the vector call protocol are now available in the :ref:`Limited API `. (Contributed by Wenzel Jakob in :gh:`98586`.) -* Added two new public functions, +* Add two new public functions, :c:func:`PyEval_SetProfileAllThreads` and :c:func:`PyEval_SetTraceAllThreads`, that allow to set tracing and profiling functions in all running threads in addition to the calling one. (Contributed by Pablo Galindo in :gh:`93503`.) -* Added new function :c:func:`PyFunction_SetVectorcall` to the C API +* Add new function :c:func:`PyFunction_SetVectorcall` to the C API which sets the vectorcall field of a given :c:type:`PyFunctionObject`. (Contributed by Andrew Frost in :gh:`92257`.) @@ -1306,14 +2043,14 @@ New Features compilers, or debuggers. (Contributed by Carl Meyer in :gh:`91052`.) -* Added :c:func:`PyType_AddWatcher` and :c:func:`PyType_Watch` API to register +* Add :c:func:`PyType_AddWatcher` and :c:func:`PyType_Watch` API to register callbacks to receive notification on changes to a type. (Contributed by Carl Meyer in :gh:`91051`.) -* Added :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` +* Add :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to register callbacks to receive notification on creation and destruction of code objects. - (Contributed by Itamar Ostricher in :gh:`91054`.) + (Contributed by Itamar Oren in :gh:`91054`.) * Add :c:func:`PyFrame_GetVar` and :c:func:`PyFrame_GetVarString` functions to get a frame variable by its name. @@ -1340,8 +2077,10 @@ New Features to replace the legacy-api :c:func:`!PyErr_Display`. (Contributed by Irit Katriel in :gh:`102755`). -* :pep:`683`: Introduced Immortal Objects to Python which allows objects - to bypass reference counts and introduced changes to the C-API: +.. _whatsnew312-pep683: + +* :pep:`683`: Introduce *Immortal Objects*, which allows objects + to bypass reference counts, and related changes to the C-API: - ``_Py_IMMORTAL_REFCNT``: The reference count that defines an object as immortal. @@ -1353,11 +2092,22 @@ New Features - ``SSTATE_INTERNED_IMMORTAL_STATIC`` An identifier for interned unicode objects that are immortal and static - ``sys.getunicodeinternedsize`` This returns the total number of unicode - objects that have been interned. This is now needed for refleak.py to + objects that have been interned. This is now needed for :file:`refleak.py` to correctly track reference counts and allocated blocks (Contributed by Eddie Elizondo in :gh:`84436`.) +* :pep:`684`: Add the new :c:func:`Py_NewInterpreterFromConfig` + function and :c:type:`PyInterpreterConfig`, which may be used + to create sub-interpreters with their own GILs. + (See :ref:`whatsnew312-pep684` for more info.) + (Contributed by Eric Snow in :gh:`104110`.) + +* In the limited C API version 3.12, :c:func:`Py_INCREF` and + :c:func:`Py_DECREF` functions are now implemented as opaque function calls to + hide implementation details. + (Contributed by Victor Stinner in :gh:`105387`.) + Porting to Python 3.12 ---------------------- @@ -1384,24 +2134,30 @@ Porting to Python 3.12 :py:meth:`~class.__subclasses__` (using :c:func:`PyObject_CallMethod`, for example). +* Add support of more formatting options (left aligning, octals, uppercase + hexadecimals, :c:type:`intmax_t`, :c:type:`ptrdiff_t`, :c:type:`wchar_t` C + strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and + :c:func:`PyUnicode_FromFormatV`. + (Contributed by Serhiy Storchaka in :gh:`98836`.) + * An unrecognized format character in :c:func:`PyUnicode_FromFormat` and :c:func:`PyUnicode_FromFormatV` now sets a :exc:`SystemError`. In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. (Contributed by Serhiy Storchaka in :gh:`95781`.) -* Fixed wrong sign placement in :c:func:`PyUnicode_FromFormat` and +* Fix wrong sign placement in :c:func:`PyUnicode_FromFormat` and :c:func:`PyUnicode_FromFormatV`. (Contributed by Philip Georgi in :gh:`95504`.) * Extension classes wanting to add a ``__dict__`` or weak reference slot - should use :const:`Py_TPFLAGS_MANAGED_DICT` and - :const:`Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and + should use :c:macro:`Py_TPFLAGS_MANAGED_DICT` and + :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and ``tp_weaklistoffset``, respectively. The use of ``tp_dictoffset`` and ``tp_weaklistoffset`` is still supported, but does not fully support multiple inheritance (:gh:`95589`), and performance may be worse. - Classes declaring :const:`Py_TPFLAGS_MANAGED_DICT` should call + Classes declaring :c:macro:`Py_TPFLAGS_MANAGED_DICT` must call :c:func:`!_PyObject_VisitManagedDict` and :c:func:`!_PyObject_ClearManagedDict` to traverse and clear their instance's dictionaries. To clear weakrefs, call :c:func:`PyObject_ClearWeakRefs`, as before. @@ -1422,7 +2178,7 @@ Porting to Python 3.12 before storing it. (Contributed by Mark Shannon in :gh:`101578`.) * ``_Py_RefTotal`` is no longer authoritative and only kept around - for ABI compabitility. Note that it is an internal global and only + for ABI compatibility. Note that it is an internal global and only available on debug builds. If you happen to be using it then you'll need to start using ``_Py_GetGlobalRefTotal()``. @@ -1439,11 +2195,67 @@ Porting to Python 3.12 allowing incomplete initialization. Note that :c:func:`PyType_FromMetaclass` (added in Python 3.12) - already disallows creating classes whose metaclass overrides ``tp_new``. + already disallows creating classes whose metaclass overrides ``tp_new`` + (:meth:`~object.__new__` in Python). + + Since ``tp_new`` overrides almost everything ``PyType_From*`` functions do, + the two are incompatible with each other. + The existing behavior -- ignoring the metaclass for several steps + of type creation -- is unsafe in general, since (meta)classes assume that + ``tp_new`` was called. + There is no simple general workaround. One of the following may work for you: + + - If you control the metaclass, avoid using ``tp_new`` in it: + + - If initialization can be skipped, it can be done in + :c:member:`~PyTypeObject.tp_init` instead. + - If the metaclass doesn't need to be instantiated from Python, + set its ``tp_new`` to ``NULL`` using + the :c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag. + This makes it acceptable for ``PyType_From*`` functions. + + - Avoid ``PyType_From*`` functions: if you don't need C-specific features + (slots or setting the instance size), create types by :ref:`calling ` + the metaclass. + + - If you *know* the ``tp_new`` can be skipped safely, filter the deprecation + warning out using :func:`warnings.catch_warnings` from Python. + +* :c:var:`PyOS_InputHook` and :c:var:`PyOS_ReadlineFunctionPointer` are no + longer called in :ref:`subinterpreters `. This is + because clients generally rely on process-wide global state (since these + callbacks have no way of recovering extension module state). + + This also avoids situations where extensions may find themselves running in a + subinterpreter that they don't support (or haven't yet been loaded in). See + :gh:`104668` for more info. + +* :c:struct:`PyLongObject` has had its internals changed for better performance. + Although the internals of :c:struct:`PyLongObject` are private, they are used + by some extension modules. + The internal fields should no longer be accessed directly, instead the API + functions beginning ``PyLong_...`` should be used instead. + Two new *unstable* API functions are provided for efficient access to the + value of :c:struct:`PyLongObject`\s which fit into a single machine word: + + * :c:func:`PyUnstable_Long_IsCompact` + * :c:func:`PyUnstable_Long_CompactValue` + +* Custom allocators, set via :c:func:`PyMem_SetAllocator`, are now + required to be thread-safe, regardless of memory domain. Allocators + that don't have their own state, including "hooks", are not affected. + If your custom allocator is not already thread-safe and you need + guidance then please create a new GitHub issue + and CC ``@ericsnowcurrently``. Deprecated ---------- +* In accordance with :pep:`699`, the ``ma_version_tag`` field in :c:type:`PyDictObject` + is deprecated for extension modules. Accessing this field will generate a compiler + warning at compile time. This field will be removed in Python 3.14. + (Contributed by Ramvikrams and Kumar Aditya in :gh:`101193`. PEP by Ken Jin.) + * Deprecate global configuration variable: * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` @@ -1465,6 +2277,7 @@ Deprecated * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` + * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` (see :c:func:`Py_PreInitialize`) @@ -1472,13 +2285,13 @@ Deprecated :c:type:`PyConfig` instead. (Contributed by Victor Stinner in :gh:`77782`.) -* Creating immutable types (:const:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable - bases is deprecated and will be disabled in Python 3.14. +* Creating :c:data:`immutable types ` with mutable + bases is deprecated and will be disabled in Python 3.14. (:gh:`95388`) -* The ``structmember.h`` header is deprecated, though it continues to be +* The :file:`structmember.h` header is deprecated, though it continues to be available and there are no plans to remove it. - Its contents are now available just by including ``Python.h``, + Its contents are now available just by including :file:`Python.h`, with a ``Py`` prefix added if it was missing: - :c:struct:`PyMemberDef`, :c:func:`PyMember_GetOne` and @@ -1488,14 +2301,14 @@ Deprecated - The flags :c:macro:`Py_READONLY` (previously ``READONLY``) and :c:macro:`Py_AUDIT_READ` (previously all uppercase) - Several items are not exposed from ``Python.h``: + Several items are not exposed from :file:`Python.h`: - :c:macro:`T_OBJECT` (use :c:macro:`Py_T_OBJECT_EX`) - :c:macro:`T_NONE` (previously undocumented, and pretty quirky) - The macro ``WRITE_RESTRICTED`` which does nothing. - The macros ``RESTRICTED`` and ``READ_RESTRICTED``, equivalents of :c:macro:`Py_AUDIT_READ`. - - In some configurations, ```` is not included from ``Python.h``. + - In some configurations, ```` is not included from :file:`Python.h`. It should be included manually when using ``offsetof()``. The deprecated header continues to provide its original @@ -1522,32 +2335,112 @@ Deprecated overrides :c:member:`~PyTypeObject.tp_new` is deprecated. Call the metaclass instead. +Pending Removal in Python 3.14 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The ``ma_version_tag`` field in :c:type:`PyDictObject` for extension modules + (:pep:`699`; :gh:`101193`). + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` + * :c:var:`Py_VerboseFlag`: use :c:member:`PyConfig.verbose` + * :c:var:`Py_QuietFlag`: use :c:member:`PyConfig.quiet` + * :c:var:`Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` + * :c:var:`Py_InspectFlag`: use :c:member:`PyConfig.inspect` + * :c:var:`Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` + * :c:var:`Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` + * :c:var:`Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` + * :c:var:`Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` + * :c:var:`Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` + * :c:var:`Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` + * :c:var:`Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` + * :c:var:`Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` + * :c:var:`Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` + * :c:var:`Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` + * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` + * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` + * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` + * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Creating :c:data:`immutable types ` with mutable + bases (:gh:`95388`). + +Pending Removal in Python 3.15 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule` +* :c:type:`!Py_UNICODE_WIDE` type: use :c:type:`wchar_t` +* :c:type:`Py_UNICODE` type: use :c:type:`wchar_t` +* Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: clear :data:`sys.warnoptions` and + :data:`!warnings.filters` + * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` + * :c:func:`Py_GetPath`: get :data:`sys.path` + * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` + * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` + * :c:func:`Py_GetProgramName`: get :data:`sys.executable` + * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or + the :envvar:`PYTHONHOME` environment variable + +Pending Removal in Future Versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following APIs are deprecated and will be removed, +although there is currently no date scheduled for their removal. + +* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: unneeded since Python 3.8 +* :c:func:`PyErr_Fetch`: use :c:func:`PyErr_GetRaisedException` +* :c:func:`PyErr_NormalizeException`: use :c:func:`PyErr_GetRaisedException` +* :c:func:`PyErr_Restore`: use :c:func:`PyErr_SetRaisedException` +* :c:func:`PyModule_GetFilename`: use :c:func:`PyModule_GetFilenameObject` +* :c:func:`PyOS_AfterFork`: use :c:func:`PyOS_AfterFork_Child` +* :c:func:`PySlice_GetIndicesEx`: use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` +* :c:func:`!PyUnicode_AsDecodedObject`: use :c:func:`PyCodec_Decode` +* :c:func:`!PyUnicode_AsDecodedUnicode`: use :c:func:`PyCodec_Decode` +* :c:func:`!PyUnicode_AsEncodedObject`: use :c:func:`PyCodec_Encode` +* :c:func:`!PyUnicode_AsEncodedUnicode`: use :c:func:`PyCodec_Encode` +* :c:func:`PyUnicode_READY`: unneeded since Python 3.12 +* :c:func:`!PyErr_Display`: use :c:func:`PyErr_DisplayException` +* :c:func:`!_PyErr_ChainExceptions`: use ``_PyErr_ChainExceptions1`` +* :c:member:`!PyBytesObject.ob_shash` member: + call :c:func:`PyObject_Hash` instead +* :c:member:`!PyDictObject.ma_version_tag` member +* Thread Local Storage (TLS) API: + + * :c:func:`PyThread_create_key`: use :c:func:`PyThread_tss_alloc` + * :c:func:`PyThread_delete_key`: use :c:func:`PyThread_tss_free` + * :c:func:`PyThread_set_key_value`: use :c:func:`PyThread_tss_set` + * :c:func:`PyThread_get_key_value`: use :c:func:`PyThread_tss_get` + * :c:func:`PyThread_delete_key_value`: use :c:func:`PyThread_tss_delete` + * :c:func:`PyThread_ReInitTLS`: unneeded since Python 3.7 + Removed ------- -* Remove the ``token.h`` header file. There was never any public tokenizer C - API. The ``token.h`` header file was only designed to be used by Python +* Remove the :file:`token.h` header file. There was never any public tokenizer C + API. The :file:`token.h` header file was only designed to be used by Python internals. (Contributed by Victor Stinner in :gh:`92651`.) * Legacy Unicode APIs have been removed. See :pep:`623` for detail. - * :c:macro:`!PyUnicode_WCHAR_KIND` - * :c:func:`!PyUnicode_AS_UNICODE` - * :c:func:`!PyUnicode_AsUnicode` - * :c:func:`!PyUnicode_AsUnicodeAndSize` - * :c:func:`!PyUnicode_AS_DATA` - * :c:func:`!PyUnicode_FromUnicode` - * :c:func:`!PyUnicode_GET_SIZE` - * :c:func:`!PyUnicode_GetSize` - * :c:func:`!PyUnicode_GET_DATA_SIZE` + * :c:macro:`!PyUnicode_WCHAR_KIND` + * :c:func:`!PyUnicode_AS_UNICODE` + * :c:func:`!PyUnicode_AsUnicode` + * :c:func:`!PyUnicode_AsUnicodeAndSize` + * :c:func:`!PyUnicode_AS_DATA` + * :c:func:`!PyUnicode_FromUnicode` + * :c:func:`!PyUnicode_GET_SIZE` + * :c:func:`!PyUnicode_GetSize` + * :c:func:`!PyUnicode_GET_DATA_SIZE` * Remove the ``PyUnicode_InternImmortal()`` function macro. (Contributed by Victor Stinner in :gh:`85858`.) - -* Remove ``Jython`` compatibility hacks from several stdlib modules and tests. - (Contributed by Nikita Sobolev in :gh:`99482`.) - -* Remove ``_use_broken_old_ctypes_structure_semantics_`` flag - from :mod:`ctypes` module. - (Contributed by Nikita Sobolev in :gh:`99285`.) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst new file mode 100644 index 00000000000000..4401deb0768c11 --- /dev/null +++ b/Doc/whatsnew/3.13.rst @@ -0,0 +1,1605 @@ + +**************************** + What's New In Python 3.13 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.13, compared to 3.12. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.13 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.13. + Brevity is key. + + +.. PEP-sized items next. + +Important deprecations, removals or restrictions: + +* :ref:`PEP 594 `: The remaining 19 "dead batteries" + have been removed from the standard library: + :mod:`!aifc`, :mod:`!audioop`, :mod:`!cgi`, :mod:`!cgitb`, :mod:`!chunk`, + :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, + :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, + :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. + +* :pep:`602` ("Annual Release Cycle for Python") has been updated: + + * Python 3.9 - 3.12 have one and a half years of full support, + followed by three and a half years of security fixes. + * Python 3.13 and later have two years of full support, + followed by three years of security fixes. + + +New Features +============ + +Improved Error Messages +----------------------- + +* The interpreter now colorizes error messages when displaying tracebacks by default. + This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment + variable as well as the canonical ``NO_COLOR`` and ``FORCE_COLOR`` environment + variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) + +Other Language Changes +====================== + +* Allow the *count* argument of :meth:`str.replace` to be a keyword. + (Contributed by Hugo van Kemenade in :gh:`106487`.) + +* Compiler now strip indents from docstrings. + This will reduce the size of :term:`bytecode cache ` (e.g. ``.pyc`` file). + For example, cache file size for ``sqlalchemy.orm.session`` in SQLAlchemy 2.0 + is reduced by about 5%. + This change will affect tools using docstrings, like :mod:`doctest`. + (Contributed by Inada Naoki in :gh:`81283`.) + +* The :func:`compile` built-in can now accept a new flag, + ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` + except that the returned ``AST`` is optimized according to the value + of the ``optimize`` argument. + (Contributed by Irit Katriel in :gh:`108113`). + +* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`: + Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the + default number of worker threads and processes. Get the CPU affinity + if supported. + (Contributed by Victor Stinner in :gh:`109649`.) + +* :func:`os.path.realpath` now resolves MS-DOS style file names even if + the file is not accessible. + (Contributed by Moonsik Park in :gh:`82367`.) + +* Fixed a bug where a :keyword:`global` declaration in an :keyword:`except` block + is rejected when the global is used in the :keyword:`else` block. + (Contributed by Irit Katriel in :gh:`111123`.) + +* Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It + determines whether or not frozen modules are ignored by the import machinery, + equivalent of the :option:`-X frozen_modules <-X>` command-line option. + (Contributed by Yilei Yang in :gh:`111374`.) + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + +ast +--- + +* :func:`ast.parse` now accepts an optional argument ``optimize`` + which is passed on to the :func:`compile` built-in. This makes it + possible to obtain an optimized ``AST``. + (Contributed by Irit Katriel in :gh:`108113`). + +array +----- + +* Add ``'w'`` type code (``Py_UCS4``) that can be used for Unicode strings. + It can be used instead of ``'u'`` type code, which is deprecated. + (Contributed by Inada Naoki in :gh:`80480`.) + +asyncio +------- + +* :meth:`asyncio.loop.create_unix_server` will now automatically remove + the Unix socket when the server is closed. + (Contributed by Pierre Ossman in :gh:`111246`.) + +copy +---- + +* Add :func:`copy.replace` function which allows to create a modified copy of + an object, which is especially useful for immutable objects. + It supports named tuples created with the factory function + :func:`collections.namedtuple`, :class:`~dataclasses.dataclass` instances, + various :mod:`datetime` objects, :class:`~inspect.Signature` objects, + :class:`~inspect.Parameter` objects, :ref:`code object `, and + any user classes which define the :meth:`!__replace__` method. + (Contributed by Serhiy Storchaka in :gh:`108751`.) + +dis +--- + +* Change the output of :mod:`dis` module functions to show logical + labels for jump targets and exception handlers, rather than offsets. + The offsets can be added with the new ``-O`` command line option or + the ``show_offsets`` parameter. + (Contributed by Irit Katriel in :gh:`112137`.) + +dbm +--- + +* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items + from the database. + (Contributed by Donghee Na in :gh:`107122`.) + +doctest +------- + +* The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped + tests. Add :attr:`doctest.DocTestRunner.skips` and + :attr:`doctest.TestResults.skipped` attributes. + (Contributed by Victor Stinner in :gh:`108794`.) + +glob +---- + +* Add :func:`glob.translate` function that converts a path specification with + shell-style wildcards to a regular expression. + (Contributed by Barney Gale in :gh:`72904`.) + +io +-- + +The :class:`io.IOBase` finalizer now logs the ``close()`` method errors with +:data:`sys.unraisablehook`. Previously, errors were ignored silently by default, +and only logged in :ref:`Python Development Mode ` or on :ref:`Python +built on debug mode `. +(Contributed by Victor Stinner in :gh:`62948`.) + +ipaddress +--------- + +* Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. + (Contributed by Charles Machalow in :gh:`109466`.) + +mmap +---- + +* The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method + that can be used where it requires a file-like object with seekable and + the :meth:`~mmap.mmap.seek` method return the new absolute position. + (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) + +opcode +------ + +* Move ``opcode.ENABLE_SPECIALIZATION`` to ``_opcode.ENABLE_SPECIALIZATION``. + This field was added in 3.12, it was never documented and is not intended for + external usage. (Contributed by Irit Katriel in :gh:`105481`.) + +* Removed ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and + ``opcode.MAX_PSEUDO_OPCODE``, which were added in 3.12, were never + documented or exposed through ``dis``, and were not intended to be + used externally. + +os +-- + +* Add :func:`os.process_cpu_count` function to get the number of logical CPUs + usable by the calling thread of the current process. + (Contributed by Victor Stinner in :gh:`109649`.) + +* Add a low level interface for Linux's timer notification file descriptors + via :func:`os.timerfd_create`, + :func:`os.timerfd_settime`, :func:`os.timerfd_settime_ns`, + :func:`os.timerfd_gettime`, and :func:`os.timerfd_gettime_ns`, + :const:`os.TFD_NONBLOCK`, :const:`os.TFD_CLOEXEC`, + :const:`os.TFD_TIMER_ABSTIME`, and :const:`os.TFD_TIMER_CANCEL_ON_SET` + (Contributed by Masaru Tsuchiyama in :gh:`108277`.) + +* :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overridden through + the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option + :option:`-X cpu_count <-X>`. This option is useful for users who need to limit + CPU resources of a container system without having to modify the container (application code). + (Contributed by Donghee Na in :gh:`109595`) + +pathlib +------- + +* Add :exc:`pathlib.UnsupportedOperation`, which is raised instead of + :exc:`NotImplementedError` when a path operation isn't supported. + (Contributed by Barney Gale in :gh:`89812`.) + +* Add :meth:`pathlib.Path.from_uri`, a new constructor to create a :class:`pathlib.Path` + object from a 'file' URI (``file:/``). + (Contributed by Barney Gale in :gh:`107465`.) + +* Add support for recursive wildcards in :meth:`pathlib.PurePath.match`. + (Contributed by Barney Gale in :gh:`73435`.) + +* Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, + :meth:`~pathlib.Path.rglob`, :meth:`~pathlib.Path.is_file`, + :meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.owner`, + :meth:`~pathlib.Path.group`. + (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and + Kamil Turek in :gh:`107962`). + +pdb +--- + +* Add ability to move between chained exceptions during post mortem debugging in :func:`~pdb.pm` using + the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias + Bussonnier in :gh:`106676`.) + +* Expressions/Statements whose prefix is a pdb command are now correctly + identified and executed. + (Contributed by Tian Gao in :gh:`108464`.) + +* ``sys.path[0]`` will no longer be replaced by the directory of the script + being debugged when ``sys.flags.safe_path`` is set (via the :option:`-P` + command line option or :envvar:`PYTHONSAFEPATH` environment variable). + (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) + +sqlite3 +------- + +* A :exc:`ResourceWarning` is now emitted if a :class:`sqlite3.Connection` + object is not :meth:`closed ` explicitly. + (Contributed by Erlend E. Aasland in :gh:`105539`.) + +sys +--- + +* Add the :func:`sys._is_interned` function to test if the string was interned. + This function is not guaranteed to exist in all implementations of Python. + (Contributed by Serhiy Storchaka in :gh:`78573`.) + +tkinter +------- + +* Add :mod:`tkinter` widget methods: + :meth:`!tk_busy_hold`, :meth:`!tk_busy_configure`, + :meth:`!tk_busy_cget`, :meth:`!tk_busy_forget`, + :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. + (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) + +* Add support of the "vsapi" element type in + the :meth:`~tkinter.ttk.Style.element_create` method of + :class:`tkinter.ttk.Style`. + (Contributed by Serhiy Storchaka in :gh:`68166`.) + +traceback +--------- + +* Add *show_group* parameter to :func:`traceback.TracebackException.format_exception_only` + to format the nested exceptions of a :exc:`BaseExceptionGroup` instance, recursively. + (Contributed by Irit Katriel in :gh:`105292`.) + +* Add the field *exc_type_str* to :class:`~traceback.TracebackException`, which + holds a string display of the *exc_type*. Deprecate the field *exc_type* + which holds the type object itself. Add parameter *save_exc_type* (default + ``True``) to indicate whether ``exc_type`` should be saved. + (Contributed by Irit Katriel in :gh:`112332`.) + +typing +------ + +* Add :func:`typing.get_protocol_members` to return the set of members + defining a :class:`typing.Protocol`. Add :func:`typing.is_protocol` to + check whether a class is a :class:`typing.Protocol`. (Contributed by Jelle Zijlstra in + :gh:`104873`.) + +unicodedata +----------- + +* The Unicode database has been updated to version 15.1.0. (Contributed by + James Gerity in :gh:`109559`.) + +venv +---- + +* Add support for adding source control management (SCM) ignore files to a + virtual environment's directory. By default, Git is supported. This is + implemented as opt-in via the API which can be extended to support other SCMs + (:class:`venv.EnvBuilder` and :func:`venv.create`), and opt-out via the CLI + (using ``--without-scm-ignore-files``). (Contributed by Brett Cannon in + :gh:`108125`.) + +warnings +-------- + +* The new :func:`warnings.deprecated` decorator provides a way to communicate + deprecations to :term:`static type checkers ` and + to warn on usage of deprecated classes and functions. A runtime deprecation + warning may also be emitted when a decorated function or class is used at runtime. + See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) + +Optimizations +============= + +* :func:`textwrap.indent` is now ~30% faster than before for large input. + (Contributed by Inada Naoki in :gh:`107369`.) + + +Deprecated +========== + +* :mod:`array`: :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, + emits :exc:`DeprecationWarning` since 3.13 + and will be removed in Python 3.16. + Use the ``'w'`` format code instead. + (contributed by Hugo van Kemenade in :gh:`80480`) + +* :mod:`ctypes`: Deprecate undocumented :func:`!ctypes.SetPointerType` + and :func:`!ctypes.ARRAY` functions. + Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. + (Contributed by Victor Stinner in :gh:`105733`.) + +* :mod:`getopt` and :mod:`optparse` modules: They are now + :term:`soft deprecated`: the :mod:`argparse` should be used for new projects. + Previously, the :mod:`optparse` module was already deprecated, its removal + was not scheduled, and no warnings was emitted: so there is no change in + practice. + (Contributed by Victor Stinner in :gh:`106535`.) + +* :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a + :exc:`DeprecationWarning` as it will be removed in 3.15. Process based CGI + http servers have been out of favor for a very long time. This code was + outdated, unmaintained, and rarely used. It has a high potential for both + security and functionality bugs. This includes removal of the ``--cgi`` + flag to the ``python -m http.server`` command line in 3.15. + +* :mod:`traceback`: + + * The field *exc_type* of :class:`traceback.TracebackException` is + deprecated. Use *exc_type_str* instead. + +* :mod:`typing`: + + * Creating a :class:`typing.NamedTuple` class using keyword arguments to denote + the fields (``NT = NamedTuple("NT", x=int, y=int)``) is deprecated, and will + be disallowed in Python 3.15. Use the class-based syntax or the functional + syntax instead. (Contributed by Alex Waygood in :gh:`105566`.) + + * When using the functional syntax to create a :class:`typing.NamedTuple` + class or a :class:`typing.TypedDict` class, failing to pass a value to the + 'fields' parameter (``NT = NamedTuple("NT")`` or ``TD = TypedDict("TD")``) is + deprecated. Passing ``None`` to the 'fields' parameter + (``NT = NamedTuple("NT", None)`` or ``TD = TypedDict("TD", None)``) is also + deprecated. Both will be disallowed in Python 3.15. To create a NamedTuple + class with 0 fields, use ``class NT(NamedTuple): pass`` or + ``NT = NamedTuple("NT", [])``. To create a TypedDict class with 0 fields, use + ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. + (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) + + * :func:`typing.no_type_check_decorator` is deprecated, and scheduled for + removal in Python 3.15. After eight years in the :mod:`typing` module, it + has yet to be supported by any major type checkers. + (Contributed by Alex Waygood in :gh:`106309`.) + + * :data:`typing.AnyStr` is deprecated. In Python 3.16, it will be removed from + ``typing.__all__``, and a :exc:`DeprecationWarning` will be emitted when it + is imported or accessed. It will be removed entirely in Python 3.18. Use + the new :ref:`type parameter syntax ` instead. + (Contributed by Michael The in :gh:`107116`.) + +* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` + methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. + They will be removed in Python 3.15. + (Contributed by Victor Stinner in :gh:`105096`.) + +* Passing more than one positional argument to :func:`sqlite3.connect` and the + :class:`sqlite3.Connection` constructor is deprecated. The remaining + parameters will become keyword-only in Python 3.15. + + Deprecate passing name, number of arguments, and the callable as keyword + arguments, for the following :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.create_function` + * :meth:`~sqlite3.Connection.create_aggregate` + + Deprecate passing the callback callable by keyword for the following + :class:`sqlite3.Connection` APIs: + + * :meth:`~sqlite3.Connection.set_authorizer` + * :meth:`~sqlite3.Connection.set_progress_handler` + * :meth:`~sqlite3.Connection.set_trace_callback` + + The affected parameters will become positional-only in Python 3.15. + + (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) + +* The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check membership + in :data:`~dis.hasarg` instead. + (Contributed by Irit Katriel in :gh:`109319`.) + +* Deprecate non-standard format specifier "N" for :class:`decimal.Decimal`. + It was not documented and only supported in the C implementation. + (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* Emit deprecation warning for non-integer numbers in :mod:`gettext` functions + and methods that consider plural forms even if the translation was not found. + (Contributed by Serhiy Storchaka in :gh:`88434`.) + +* Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` + (as has always been the case for an executing frame). + (Contributed by Irit Katriel in :gh:`79932`.) + +* Assignment to a function's ``__code__`` attribute where the new code + object's type does not match the function's type, is deprecated. The + different types are: plain function, generator, async generator and + coroutine. + (Contributed by Irit Katriel in :gh:`81137`.) + + +Pending Removal in Python 3.14 +------------------------------ + +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are deprecated + and will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`92248`.) + +* :mod:`ast`: The following features have been deprecated in documentation + since Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at + runtime when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + +* :mod:`collections.abc`: Deprecated :class:`~collections.abc.ByteString`. + Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, + or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. + (Contributed by Alan Williams in :gh:`72346`.) + +* :mod:`importlib`: ``__package__`` and ``__cached__`` will cease to be set or + taken into consideration by the import system (:gh:`97879`). + +* :mod:`importlib.abc` deprecated classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +* :mod:`itertools` had undocumented, inefficient, historically buggy, + and inconsistent support for copy, deepcopy, and pickle operations. + This will be removed in 3.14 for a significant reduction in code + volume and maintenance burden. + (Contributed by Raymond Hettinger in :gh:`101588`.) + +* :mod:`multiprocessing`: The default start method will change to a safer one on + Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently + the default (:gh:`84559`). Adding a runtime warning about this was deemed too + disruptive as the majority of code is not expected to care. Use the + :func:`~multiprocessing.get_context` or + :func:`~multiprocessing.set_start_method` APIs to explicitly specify when + your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. + +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to`, + :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is + deprecated. + +* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` + now raise :exc:`DeprecationWarning`; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + +* :mod:`pty`: + + * ``master_open()``: use :func:`pty.openpty`. + * ``slave_open()``: use :func:`pty.openpty`. + +* :func:`shutil.rmtree` *onerror* parameter is deprecated in 3.12, + and will be removed in 3.14: use the *onexc* parameter instead. + +* :mod:`sqlite3`: + + * :data:`~sqlite3.version` and :data:`~sqlite3.version_info`. + + * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` + if :ref:`named placeholders ` are used and + *parameters* is a sequence instead of a :class:`dict`. + + * date and datetime adapter, date and timestamp converter: + see the :mod:`sqlite3` documentation for suggested replacement recipes. + +* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` + since 3.10 and was planned to be removed in 3.12, + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`101866`.) + +* :mod:`typing`: :class:`~typing.ByteString`, deprecated since Python 3.9, + now causes a :exc:`DeprecationWarning` to be emitted when it is used. + +* :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a + public API. + (Contributed by Gregory P. Smith in :gh:`88168`.) + +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`~xml.etree.ElementTree.Element` is deprecated and will raise an + exception in Python 3.14. + +Pending Removal in Python 3.15 +------------------------------ + +* :class:`http.server.CGIHTTPRequestHandler` will be removed along with its + related ``--cgi`` flag to ``python -m http.server``. It was obsolete and + rarely used. No direct replacement exists. *Anything* is better than CGI + to interface a web server with a request handler. + +* :class:`locale`: :func:`locale.getdefaultlocale` was deprecated in Python 3.11 + and originally planned for removal in Python 3.13 (:gh:`90817`), + but removal has been postponed to Python 3.15. + Use :func:`locale.setlocale()`, :func:`locale.getencoding()` and + :func:`locale.getlocale()` instead. + (Contributed by Hugo van Kemenade in :gh:`111187`.) + +* :class:`typing.NamedTuple`: + + * The undocumented keyword argument syntax for creating NamedTuple classes + (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed in + 3.15. Use the class-based syntax or the functional syntax instead. + + * When using the functional syntax to create a NamedTuple class, failing to + pass a value to the 'fields' parameter (``NT = NamedTuple("NT")``) is + deprecated. Passing ``None`` to the 'fields' parameter + (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be + disallowed in Python 3.15. To create a NamedTuple class with 0 fields, use + ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. + +* :class:`typing.TypedDict`: When using the functional syntax to create a + TypedDict class, failing to pass a value to the 'fields' parameter (``TD = + TypedDict("TD")``) is deprecated. Passing ``None`` to the 'fields' parameter + (``TD = TypedDict("TD", None)``) is also deprecated. Both will be disallowed + in Python 3.15. To create a TypedDict class with 0 fields, use ``class + TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. + +* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` + methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. + They will be removed in Python 3.15. + (Contributed by Victor Stinner in :gh:`105096`.) + +* Passing any arguments to :func:`threading.RLock` is now deprecated. + C version allows any numbers of args and kwargs, + but they are just ignored. Python version does not allow any arguments. + All arguments will be removed from :func:`threading.RLock` in Python 3.15. + (Contributed by Nikita Sobolev in :gh:`102029`.) + +Pending Removal in Python 3.16 +------------------------------ + +* :class:`array.array` ``'u'`` type (:c:type:`wchar_t`): + use the ``'w'`` type instead (``Py_UCS4``). + +Pending Removal in Future Versions +---------------------------------- + +The following APIs were deprecated in earlier Python versions and will be removed, +although there is currently no date scheduled for their removal. + +* :mod:`argparse`: Nesting argument groups and nesting mutually exclusive + groups are deprecated. + +* :mod:`builtins`: + + * ``~bool``, bitwise inversion on bool. + * ``bool(NotImplemented)``. + * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` + signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, + the single argument signature. + * Currently Python accepts numeric literals immediately followed by keywords, + for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and + ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as + ``[0x1 for x in y]`` or ``[0x1f or x in y]``). A syntax warning is raised + if the numeric literal is immediately followed by one of keywords + :keyword:`and`, :keyword:`else`, :keyword:`for`, :keyword:`if`, + :keyword:`in`, :keyword:`is` and :keyword:`or`. In a future release it + will be changed to a syntax error. (:gh:`87999`) + * Support for ``__index__()`` and ``__int__()`` method returning non-int type: + these methods will be required to return an instance of a strict subclass of + :class:`int`. + * Support for ``__float__()`` method returning a strict subclass of + :class:`float`: these methods will be required to return an instance of + :class:`float`. + * Support for ``__complex__()`` method returning a strict subclass of + :class:`complex`: these methods will be required to return an instance of + :class:`complex`. + * Delegation of ``int()`` to ``__trunc__()`` method. + +* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are + deprecated and replaced by :data:`calendar.JANUARY` and + :data:`calendar.FEBRUARY`. + (Contributed by Prince Roshan in :gh:`103636`.) + +* :mod:`datetime`: + + * :meth:`~datetime.datetime.utcnow`: + use ``datetime.datetime.now(tz=datetime.UTC)``. + * :meth:`~datetime.datetime.utcfromtimestamp`: + use ``datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)``. + +* :mod:`gettext`: Plural value must be an integer. + +* :mod:`importlib`: + + * ``load_module()`` method: use ``exec_module()`` instead. + * :func:`~importlib.util.cache_from_source` *debug_override* parameter is + deprecated: use the *optimization* parameter instead. + +* :mod:`importlib.metadata`: + + * ``EntryPoints`` tuple interface. + * Implicit ``None`` on return values. + +* :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use + BytesIO and binary mode instead. + +* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. + +* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is + deprecated, use an exception instance. + +* :mod:`re`: More strict rules are now applied for numerical group references + and group names in regular expressions. Only sequence of ASCII digits is now + accepted as a numerical reference. The group name in bytes patterns and + replacement strings can now only contain ASCII letters and digits and + underscore. + (Contributed by Serhiy Storchaka in :gh:`91760`.) + +* :mod:`ssl` options and protocols: + + * :class:`ssl.SSLContext` without protocol argument is deprecated. + * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and + :meth:`!~ssl.SSLContext.selected_npn_protocol` are deprecated: use ALPN + instead. + * ``ssl.OP_NO_SSL*`` options + * ``ssl.OP_NO_TLS*`` options + * ``ssl.PROTOCOL_SSLv3`` + * ``ssl.PROTOCOL_TLS`` + * ``ssl.PROTOCOL_TLSv1`` + * ``ssl.PROTOCOL_TLSv1_1`` + * ``ssl.PROTOCOL_TLSv1_2`` + * ``ssl.TLSVersion.SSLv3`` + * ``ssl.TLSVersion.TLSv1`` + * ``ssl.TLSVersion.TLSv1_1`` + +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + +* :attr:`~codeobject.co_lnotab`: use the ``co_lines`` attribute instead. + +* :class:`typing.Text` (:gh:`92332`). + +* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and + ignored. + +* :mod:`threading` methods: + + * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. + * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. + * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: + use :attr:`threading.Thread.daemon` attribute. + * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: + use :attr:`threading.Thread.name` attribute. + * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. + * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. + +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not None from a test case. + +* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and + :class:`~urllib.request.FancyURLopener` style of invoking requests is + deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. + +* :func:`!urllib.parse.to_bytes`. + +* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead + + * ``splitattr()`` + * ``splithost()`` + * ``splitnport()`` + * ``splitpasswd()`` + * ``splitport()`` + * ``splitquery()`` + * ``splittag()`` + * ``splittype()`` + * ``splituser()`` + * ``splitvalue()`` + +* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial + writes. + +* :meth:`zipimport.zipimporter.load_module` is deprecated: + use :meth:`~zipimport.zipimporter.exec_module` instead. + + +Removed +======= + +.. _whatsnew313-pep594: + +PEP 594: dead batteries +----------------------- + +* :pep:`594` removed 19 modules from the standard library, + deprecated in Python 3.11: + + * :mod:`!aifc`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!audioop`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!chunk`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!cgi` and :mod:`!cgitb`. + + * ``cgi.FieldStorage`` can typically be replaced with + :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, + and the :mod:`email.message` module or `multipart + `__ PyPI project for ``POST`` and + ``PUT``. + + * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` + directly on the desired query string, except for ``multipart/form-data`` + input, which can be handled as described for ``cgi.parse_multipart()``. + + * ``cgi.parse_header()`` can be replaced with the functionality in the + :mod:`email` package, which implements the same MIME RFCs. For example, + with :class:`email.message.EmailMessage`:: + + from email.message import EmailMessage + msg = EmailMessage() + msg['content-type'] = 'application/json; charset="utf8"' + main, params = msg.get_content_type(), msg['content-type'].params + + * ``cgi.parse_multipart()`` can be replaced with the functionality in the + :mod:`email` package (e.g. :class:`email.message.EmailMessage` and + :class:`email.message.Message`) which implements the same MIME RFCs, or + with the `multipart `__ PyPI project. + + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!crypt` module and its private :mod:`!_crypt` extension. + The :mod:`hashlib` module is a potential replacement for certain use cases. + Otherwise, the following PyPI projects can be used: + + * `bcrypt `_: + Modern password hashing for your software and your servers. + * `passlib `_: + Comprehensive password hashing framework supporting over 30 schemes. + * `argon2-cffi `_: + The secure Argon2 password hashing algorithm. + * `legacycrypt `_: + Wrapper to the POSIX crypt library call and associated functionality. + + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!imghdr`: use the projects + `filetype `_, + `puremagic `_, + or `python-magic `_ instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!mailcap`. + The :mod:`mimetypes` module provides an alternative. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!msilib`. + (Contributed by Zachary Ware in :gh:`104773`.) + + * :mod:`!nis`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!nntplib`: + the `PyPI nntplib project `_ + can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!ossaudiodev`: use the + `pygame project `_ for audio playback. + (Contributed by Victor Stinner in :gh:`104780`.) + + * :mod:`!pipes`: use the :mod:`subprocess` module instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!sndhdr`: use the projects + `filetype `_, + `puremagic `_, or + `python-magic `_ instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!spwd`: + the `python-pam project `_ + can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!sunau`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!telnetlib`, use the projects + `telnetlib3 `_ or + `Exscript `_ instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!uu`: the :mod:`base64` module is a modern alternative. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!xdrlib`. + (Contributed by Victor Stinner in :gh:`104773`.) + +2to3 +---- + +* Remove the ``2to3`` program and the :mod:`!lib2to3` module, + deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104780`.) + +configparser +------------ + +* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, + deprecated in the docstring since Python 3.2, + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104886`.) + +importlib +--------- + +* Remove :mod:`importlib.resources` deprecated methods: + + * ``contents()`` + * ``is_resource()`` + * ``open_binary()`` + * ``open_text()`` + * ``path()`` + * ``read_binary()`` + * ``read_text()`` + + Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy + `_ + for migration advice. + (Contributed by Jason R. Coombs in :gh:`106532`.) + +locale +------ + +* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: + use ``locale.setlocale(locale.LC_ALL, "")`` instead. + (Contributed by Victor Stinner in :gh:`104783`.) + +logging +------- + +* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and + ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated + since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` + method, :meth:`!logging.LoggerAdapter.warning` method and + :func:`logging.warning` function. + (Contributed by Victor Stinner in :gh:`105376`.) + +pathlib +------- + +* Remove support for using :class:`pathlib.Path` objects as context managers. + This functionality was deprecated and made a no-op in Python 3.9. + +re +-- + +* Remove undocumented, never working, and deprecated ``re.template`` function + and ``re.TEMPLATE`` flag (and ``re.T`` alias). + (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) + +tkinter +------- + +* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The + third-party Tix library which the module wrapped is unmaintained. + (Contributed by Zachary Ware in :gh:`75552`.) + +turtle +------ + +* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, + deprecated in docs since Python 3.1 + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104876`.) + +typing +------ + +* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, + are now removed. The items in those namespaces can be imported directly + from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) + +* Remove support for the keyword-argument method of creating + :class:`typing.TypedDict` types, deprecated in Python 3.11. + (Contributed by Tomas Roun in :gh:`104786`.) + +unittest +-------- + +* Removed the following :mod:`unittest` functions, deprecated in Python 3.11: + + * :func:`!unittest.findTestCases` + * :func:`!unittest.makeSuite` + * :func:`!unittest.getTestCaseNames` + + Use :class:`~unittest.TestLoader` methods instead: + + * :meth:`unittest.TestLoader.loadTestsFromModule` + * :meth:`unittest.TestLoader.loadTestsFromTestCase` + * :meth:`unittest.TestLoader.getTestCaseNames` + + (Contributed by Hugo van Kemenade in :gh:`104835`.) + +* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` + method, deprecated in Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104992`.) + +urllib +------ + +* Remove *cafile*, *capath* and *cadefault* parameters of the + :func:`urllib.request.urlopen` function, deprecated in Python 3.6: use the + *context* parameter instead. Please use + :meth:`ssl.SSLContext.load_cert_chain` instead, or let + :func:`ssl.create_default_context` select the system's trusted CA + certificates for you. + (Contributed by Victor Stinner in :gh:`105382`.) + +webbrowser +---------- + +* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, + deprecated in Python 3.11. + Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. + (Contributed by Hugo van Kemenade in :gh:`104804`.) + +* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. + Use :attr:`webbrowser.MacOSXOSAScript.name ` + attribute instead. + (Contributed by Nikita Sobolev in :gh:`105546`.) + +Others +------ + +* None yet + +CPython bytecode changes +======================== + +* The oparg of ``YIELD_VALUE`` is now ``1`` if the yield is part of a + yield-from or await, and ``0`` otherwise. The oparg of ``RESUME`` was + changed to add a bit indicating whether the except-depth is 1, which + is needed to optimize closing of generators. + (Contributed by Irit Katriel in :gh:`111354`.) + +Porting to Python 3.13 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +* :meth:`!tkinter.Text.count` now always returns an integer if one or less + counting options are specified. + Previously it could return a single count as a 1-tuple, an integer (only if + option ``"update"`` was specified) or ``None`` if no items found. + The result is now the same if ``wantobjects`` is set to ``0``. + (Contributed by Serhiy Storchaka in :gh:`97928`.) + +* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, + :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, + :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and + :c:func:`PySys_GetObject`, which clear all errors occurred during calling + the function, report now them using :func:`sys.unraisablehook`. + You can consider to replace these functions with other functions as + recomended in the documentation. + (Contributed by Serhiy Storchaka in :gh:`106672`.) + +* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to + retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or + :exc:`KeyError` on Unix platforms where the password database is empty. + +* The :mod:`threading` module now expects the :mod:`!_thread` module to have + an ``_is_main_interpreter`` attribute. It is a function with no + arguments that returns ``True`` if the current interpreter is the + main interpreter. + + Any library or application that provides a custom ``_thread`` module + must provide ``_is_main_interpreter()``, just like the module's + other "private" attributes. + (See :gh:`112826`.) + + +Build Changes +============= + +* Autoconf 2.71 and aclocal 1.16.4 is now required to regenerate + the :file:`configure` script. + (Contributed by Christian Heimes in :gh:`89886`.) + +* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. + (Contributed by Erlend Aasland in :gh:`105875`.) + +* Python built with :file:`configure` :option:`--with-trace-refs` (tracing + references) is now ABI compatible with Python release build and + :ref:`debug build `. + (Contributed by Victor Stinner in :gh:`108634`.) + +* Building CPython now requires a compiler with support for the C11 atomic + library, GCC built-in atomic functions, or MSVC interlocked intrinsics. + +* The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, + ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, + ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the + :ref:`limited C API `. + (Contributed by Victor Stinner in :gh:`85283`.) + + +C API Changes +============= + +New Features +------------ + +* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before including + :file:`Python.h` when using ``#`` formats in + :ref:`format codes `. + APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. + (Contributed by Inada Naoki in :gh:`104922`.) + +* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and + :c:func:`PyArg_VaParseTupleAndKeywords` has now type :c:expr:`char * const *` + in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. + It makes these functions compatible with arguments of type + :c:expr:`const char * const *`, :c:expr:`const char **` or + :c:expr:`char * const *` in C++ and :c:expr:`char * const *` in C + without an explicit type cast. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + (Contributed by Serhiy Storchaka in :gh:`65210`.) + +* Add :c:func:`PyImport_AddModuleRef`: similar to + :c:func:`PyImport_AddModule`, but return a :term:`strong reference` instead + of a :term:`borrowed reference`. + (Contributed by Victor Stinner in :gh:`105922`.) + +* Add :c:func:`PyWeakref_GetRef` function: similar to + :c:func:`PyWeakref_GetObject` but returns a :term:`strong reference`, or + ``NULL`` if the referent is no longer live. + (Contributed by Victor Stinner in :gh:`105927`.) + +* Add :c:func:`PyObject_GetOptionalAttr` and + :c:func:`PyObject_GetOptionalAttrString`, variants of + :c:func:`PyObject_GetAttr` and :c:func:`PyObject_GetAttrString` which + don't raise :exc:`AttributeError` if the attribute is not found. + These variants are more convenient and faster if the missing attribute + should not be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106521`.) + +* Add :c:func:`PyMapping_GetOptionalItem` and + :c:func:`PyMapping_GetOptionalItemString`: variants of + :c:func:`PyObject_GetItem` and :c:func:`PyMapping_GetItemString` which don't + raise :exc:`KeyError` if the key is not found. + These variants are more convenient and faster if the missing key should not + be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106307`.) + +* Add fixed variants of functions which silently ignore errors: + + - :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. + - :c:func:`PyObject_HasAttrStringWithError` replaces :c:func:`PyObject_HasAttrString`. + - :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. + - :c:func:`PyMapping_HasKeyStringWithError` replaces :c:func:`PyMapping_HasKeyString`. + + New functions return not only ``1`` for true and ``0`` for false, but also + ``-1`` for error. + + (Contributed by Serhiy Storchaka in :gh:`108511`.) + +* If Python is built in :ref:`debug mode ` or :option:`with + assertions <--with-assertions>`, :c:func:`PyTuple_SET_ITEM` and + :c:func:`PyList_SET_ITEM` now check the index argument with an assertion. + (Contributed by Victor Stinner in :gh:`106168`.) + +* Add :c:func:`PyModule_Add` function: similar to + :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject` but + always steals a reference to the value. + (Contributed by Serhiy Storchaka in :gh:`86493`.) + +* Added :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` + functions: similar to :c:func:`PyDict_GetItemWithError` but returning a + :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, + these functions return -1 on error and so checking ``PyErr_Occurred()`` is + not needed. + (Contributed by Victor Stinner in :gh:`106004`.) + +* Added :c:func:`PyDict_ContainsString` function: same as + :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` + UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. + (Contributed by Victor Stinner in :gh:`108314`.) + +* Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is + :term:`shutting down `. + (Contributed by Victor Stinner in :gh:`108014`.) + +* Add :c:func:`PyLong_AsInt` function: similar to :c:func:`PyLong_AsLong`, but + store the result in a C :c:expr:`int` instead of a C :c:expr:`long`. + Previously, it was known as the private function :c:func:`!_PyLong_AsInt` + (with an underscore prefix). + (Contributed by Victor Stinner in :gh:`108014`.) + +* Python built with :file:`configure` :option:`--with-trace-refs` (tracing + references) now supports the :ref:`Limited API `. + (Contributed by Victor Stinner in :gh:`108634`.) + +* Add :c:func:`PyObject_VisitManagedDict` and + :c:func:`PyObject_ClearManagedDict` functions which must be called by the + traverse and clear functions of a type using + :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. The `pythoncapi-compat project + `__ can be used to get these + functions on Python 3.11 and 3.12. + (Contributed by Victor Stinner in :gh:`107073`.) + +* Add :c:func:`PyUnicode_EqualToUTF8AndSize` and :c:func:`PyUnicode_EqualToUTF8` + functions: compare Unicode object with a :c:expr:`const char*` UTF-8 encoded + string and return true (``1``) if they are equal, or false (``0``) otherwise. + These functions do not raise exceptions. + (Contributed by Serhiy Storchaka in :gh:`110289`.) + +* Add :c:func:`PyThreadState_GetUnchecked()` function: similar to + :c:func:`PyThreadState_Get()`, but don't kill the process with a fatal error + if it is NULL. The caller is responsible to check if the result is NULL. + Previously, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + (Contributed by Victor Stinner in :gh:`108867`.) + +* Add :c:func:`PySys_AuditTuple` function: similar to :c:func:`PySys_Audit`, + but pass event arguments as a Python :class:`tuple` object. + (Contributed by Victor Stinner in :gh:`85283`.) + +* :c:func:`PyArg_ParseTupleAndKeywords` now supports non-ASCII keyword + parameter names. + (Contributed by Serhiy Storchaka in :gh:`110815`.) + +* Add :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawCalloc`, + :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` to the limited C API + (version 3.13). + (Contributed by Victor Stinner in :gh:`85283`.) + +* Add :c:func:`PySys_Audit` and :c:func:`PySys_AuditTuple` functions to the + limited C API. + (Contributed by Victor Stinner in :gh:`85283`.) + +* Add :c:func:`PyErr_FormatUnraisable` function: similar to + :c:func:`PyErr_WriteUnraisable`, but allow to customize the warning mesage. + (Contributed by Serhiy Storchaka in :gh:`108082`.) + +* Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to + Python ``list.extend()`` and ``list.clear()`` methods. + (Contributed by Victor Stinner in :gh:`111138`.) + +* Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a + key from a dictionary and optionally return the removed value. This is + similar to :meth:`dict.pop`, but without the default value and not raising + :exc:`KeyError` if the key missing. + (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) + +* Add :c:func:`Py_HashPointer` function to hash a pointer. + (Contributed by Victor Stinner in :gh:`111545`.) + + +Porting to Python 3.13 +---------------------- + +* ``Python.h`` no longer includes the ```` standard header. It was + included for the ``finite()`` function which is now provided by the + ```` header. It should now be included explicitly if needed. Remove + also the ``HAVE_IEEEFP_H`` macro. + (Contributed by Victor Stinner in :gh:`108765`.) + +* ``Python.h`` no longer includes these standard header files: ````, + ```` and ````. If needed, they should now be + included explicitly. For example, ```` provides the ``clock()`` and + ``gmtime()`` functions, ```` provides the ``select()`` + function, and ```` provides the ``futimes()``, ``gettimeofday()`` + and ``setitimer()`` functions. + (Contributed by Victor Stinner in :gh:`108765`.) + +* If the :c:macro:`Py_LIMITED_API` macro is defined, :c:macro:`!Py_BUILD_CORE`, + :c:macro:`!Py_BUILD_CORE_BUILTIN` and :c:macro:`!Py_BUILD_CORE_MODULE` macros + are now undefined by ````. + (Contributed by Victor Stinner in :gh:`85283`.) + +* The old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and ``Py_TRASHCAN_SAFE_END`` + were removed. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` + and ``Py_TRASHCAN_END``. + + A tp_dealloc function that has the old macros, such as:: + + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_SAFE_BEGIN(p); + ... + Py_TRASHCAN_SAFE_END + } + + should migrate to the new macros as follows:: + + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_BEGIN(p, mytype_dealloc) + ... + Py_TRASHCAN_END + } + + Note that ``Py_TRASHCAN_BEGIN`` has a second argument which + should be the deallocation function it is in. + +* On Windows, ``Python.h`` no longer includes the ```` standard + header file. If needed, it should now be included explicitly. For example, it + provides ``offsetof()`` function, and ``size_t`` and ``ptrdiff_t`` types. + Including ```` explicitly was already needed by all other + platforms, the ``HAVE_STDDEF_H`` macro is only defined on Windows. + (Contributed by Victor Stinner in :gh:`108765`.) + + +Deprecated +---------- + +* Passing optional arguments *maxsplit*, *count* and *flags* in module-level + functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional + arguments is now deprecated. + In future Python versions these parameters will be + :ref:`keyword-only `. + (Contributed by Serhiy Storchaka in :gh:`56166`.) + +* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly + the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and + ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. + (Contributed by Victor Stinner in :gh:`105156`.) + +* Deprecate old Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: + clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or + :envvar:`PYTHONHOME` environment variable instead. + + Functions scheduled for removal in Python 3.15. + (Contributed by Victor Stinner in :gh:`105145`.) + +* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just + an alias to :c:func:`PyImport_ImportModule` since Python 3.3. + Scheduled for removal in Python 3.15. + (Contributed by Victor Stinner in :gh:`105396`.) + +* Deprecate the :c:func:`PyWeakref_GetObject` and + :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed + reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it + returns a :term:`strong reference`. The `pythoncapi-compat project + `__ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Victor Stinner in :gh:`105927`.) + +Removed +------- + +* Removed chained :class:`classmethod` descriptors (introduced in + :issue:`19072`). This can no longer be used to wrap other descriptors + such as :class:`property`. The core design of this feature was flawed + and caused a number of downstream problems. To "pass-through" a + :class:`classmethod`, consider using the :attr:`!__wrapped__` + attribute that was added in Python 3.10. (Contributed by Raymond + Hettinger in :gh:`89519`.) + +* Remove many APIs (functions, macros, variables) with names prefixed by + ``_Py`` or ``_PY`` (considered as private API). If your project is affected + by one of these removals and you consider that the removed API should remain + available, please open a new issue to request a public C API and + add ``cc @vstinner`` to the issue to notify Victor Stinner. + (Contributed by Victor Stinner in :gh:`106320`.) + +* Remove functions deprecated in Python 3.9. + + * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use + :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. + Warning: :c:func:`PyObject_Call` positional arguments must be a + :class:`tuple` and must not be *NULL*, keyword arguments must be a + :class:`dict` or *NULL*, whereas removed functions checked arguments type + and accepted *NULL* positional and keyword arguments. + To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with + :c:func:`PyObject_Call`, pass an empty tuple as positional arguments using + :c:func:`PyTuple_New(0) `. + * ``PyEval_CallFunction()``: use :c:func:`PyObject_CallFunction` instead. + * ``PyEval_CallMethod()``: use :c:func:`PyObject_CallMethod` instead. + * ``PyCFunction_Call()``: use :c:func:`PyObject_Call` instead. + + (Contributed by Victor Stinner in :gh:`105107`.) + +* Remove old buffer protocols deprecated in Python 3.0. Use :ref:`bufferobjects` instead. + + * :c:func:`!PyObject_CheckReadBuffer`: Use :c:func:`PyObject_CheckBuffer` to + test if the object supports the buffer protocol. + Note that :c:func:`PyObject_CheckBuffer` doesn't guarantee that + :c:func:`PyObject_GetBuffer` will succeed. + To test if the object is actually readable, see the next example + of :c:func:`PyObject_GetBuffer`. + + * :c:func:`!PyObject_AsCharBuffer`, :c:func:`!PyObject_AsReadBuffer`: + :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: + + .. code-block:: c + + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) < 0) { + return NULL; + } + // Use `view.buf` and `view.len` to read from the buffer. + // You may need to cast buf as `(const char*)view.buf`. + PyBuffer_Release(&view); + + * :c:func:`!PyObject_AsWriteBuffer`: Use + :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: + + .. code-block:: c + + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_WRITABLE) < 0) { + return NULL; + } + // Use `view.buf` and `view.len` to write to the buffer. + PyBuffer_Release(&view); + + (Contributed by Inada Naoki in :gh:`85275`.) + +* Remove the following old functions to configure the Python initialization, + deprecated in Python 3.11: + + * ``PySys_AddWarnOptionUnicode()``: use :c:member:`PyConfig.warnoptions` instead. + * ``PySys_AddWarnOption()``: use :c:member:`PyConfig.warnoptions` instead. + * ``PySys_AddXOption()``: use :c:member:`PyConfig.xoptions` instead. + * ``PySys_HasWarnOptions()``: use :c:member:`PyConfig.xoptions` instead. + * ``PySys_SetArgvEx()``: set :c:member:`PyConfig.argv` instead. + * ``PySys_SetArgv()``: set :c:member:`PyConfig.argv` instead. + * ``PySys_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. + * ``Py_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. + * ``Py_SetProgramName()``: set :c:member:`PyConfig.program_name` instead. + * ``Py_SetPythonHome()``: set :c:member:`PyConfig.home` instead. + * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` + instead, and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on + Windows). + * ``_Py_SetProgramFullPath()``: set :c:member:`PyConfig.executable` instead. + + Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization + Configuration ` instead (:pep:`587`), added to Python 3.8. + (Contributed by Victor Stinner in :gh:`105145`.) + +* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and + ``Py_TRASHCAN_SAFE_END``. They should be replaced by the new macros + ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. The new macros were + added in Python 3.8 and the old macros were deprecated in Python 3.11. + (Contributed by Irit Katriel in :gh:`105111`.) + +* Remove ``PyEval_InitThreads()`` and ``PyEval_ThreadsInitialized()`` + functions, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` + always creates the GIL: calling ``PyEval_InitThreads()`` did nothing and + ``PyEval_ThreadsInitialized()`` always returned non-zero. + (Contributed by Victor Stinner in :gh:`105182`.) + +* Remove ``PyEval_AcquireLock()`` and ``PyEval_ReleaseLock()`` functions, + deprecated in Python 3.2. They didn't update the current thread state. + They can be replaced with: + + * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`; + * low-level :c:func:`PyEval_AcquireThread` and :c:func:`PyEval_RestoreThread`; + * or :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. + + (Contributed by Victor Stinner in :gh:`105182`.) + +* Remove private ``_PyObject_FastCall()`` function: + use ``PyObject_Vectorcall()`` which is available since Python 3.8 + (:pep:`590`). + (Contributed by Victor Stinner in :gh:`106023`.) + +* Remove ``cpython/pytime.h`` header file: it only contained private functions. + (Contributed by Victor Stinner in :gh:`106316`.) + +* Remove ``_PyInterpreterState_Get()`` alias to + :c:func:`PyInterpreterState_Get()` which was kept for backward compatibility + with Python 3.8. The `pythoncapi-compat project + `__ can be used to get + :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. + (Contributed by Victor Stinner in :gh:`106320`.) + +* The :c:func:`PyModule_AddObject` function is now :term:`soft deprecated`: + :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` functions should + be used instead. + (Contributed by Serhiy Storchaka in :gh:`86493`.) + +Pending Removal in Python 3.14 +------------------------------ + +* Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable + bases using the C API. +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` + * :c:var:`Py_VerboseFlag`: use :c:member:`PyConfig.verbose` + * :c:var:`Py_QuietFlag`: use :c:member:`PyConfig.quiet` + * :c:var:`Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` + * :c:var:`Py_InspectFlag`: use :c:member:`PyConfig.inspect` + * :c:var:`Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` + * :c:var:`Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` + * :c:var:`Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` + * :c:var:`Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` + * :c:var:`Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` + * :c:var:`Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` + * :c:var:`Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` + * :c:var:`Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` + * :c:var:`Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` + * :c:var:`Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` + * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` + * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` + * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` + * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +Pending Removal in Python 3.15 +------------------------------ + +* :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule`. +* :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead. +* :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead. +* :c:type:`!Py_UNICODE_WIDE` type: use :c:type:`wchar_t` instead. +* :c:type:`Py_UNICODE` type: use :c:type:`wchar_t` instead. +* Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: clear :data:`sys.warnoptions` and + :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or + :envvar:`PYTHONHOME` environment variable instead. + +Pending Removal in Future Versions +---------------------------------- + +The following APIs were deprecated in earlier Python versions and will be +removed, although there is currently no date scheduled for their removal. + +* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: no needed since Python 3.8. +* :c:func:`PyErr_Fetch`: use :c:func:`PyErr_GetRaisedException`. +* :c:func:`PyErr_NormalizeException`: use :c:func:`PyErr_GetRaisedException`. +* :c:func:`PyErr_Restore`: use :c:func:`PyErr_SetRaisedException`. +* :c:func:`PyModule_GetFilename`: use :c:func:`PyModule_GetFilenameObject`. +* :c:func:`PyOS_AfterFork`: use :c:func:`PyOS_AfterFork_Child()`. +* :c:func:`PySlice_GetIndicesEx`. +* :c:func:`!PyUnicode_AsDecodedObject`. +* :c:func:`!PyUnicode_AsDecodedUnicode`. +* :c:func:`!PyUnicode_AsEncodedObject`. +* :c:func:`!PyUnicode_AsEncodedUnicode`. +* :c:func:`PyUnicode_READY`: not needed since Python 3.12. +* :c:func:`!_PyErr_ChainExceptions`. +* :c:member:`!PyBytesObject.ob_shash` member: + call :c:func:`PyObject_Hash` instead. +* :c:member:`!PyDictObject.ma_version_tag` member. +* TLS API: + + * :c:func:`PyThread_create_key`: use :c:func:`PyThread_tss_alloc`. + * :c:func:`PyThread_delete_key`: use :c:func:`PyThread_tss_free`. + * :c:func:`PyThread_set_key_value`: use :c:func:`PyThread_tss_set`. + * :c:func:`PyThread_get_key_value`: use :c:func:`PyThread_tss_get`. + * :c:func:`PyThread_delete_key_value`: use :c:func:`PyThread_tss_delete`. + * :c:func:`PyThread_ReInitTLS`: no longer needed. + +* Remove undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. + (Contributed by Victor Stinner in :gh:`110014`.) + + +Regression Test Changes +======================= + +* Python built with :file:`configure` :option:`--with-pydebug` now + supports a :option:`-X presite=package.module <-X>` command-line + option. If used, it specifies a module that should be imported early + in the lifecycle of the interpreter, before ``site.py`` is executed. + (Contributed by Åukasz Langa in :gh:`110769`.) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 7af0c0288376fd..df32b76b6d7b03 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -424,7 +424,7 @@ protocols, the users must to be able access the environment using native strings even though the underlying platform may have a different convention. To bridge this gap, the :mod:`wsgiref` module has a new function, :func:`wsgiref.handlers.read_environ` for transcoding CGI variables from -:attr:`os.environ` into native strings and returning a new dictionary. +:data:`os.environ` into native strings and returning a new dictionary. .. seealso:: @@ -485,7 +485,7 @@ Some smaller changes made to the core Python language are: * The interpreter can now be started with a quiet option, ``-q``, to prevent the copyright and version information from being displayed in the interactive - mode. The option can be introspected using the :attr:`sys.flags` attribute: + mode. The option can be introspected using the :data:`sys.flags` attribute: .. code-block:: shell-session @@ -566,9 +566,9 @@ Some smaller changes made to the core Python language are: (See :issue:`4617`.) -* The internal :c:type:`structsequence` tool now creates subclasses of tuple. +* :ref:`Struct sequence types ` are now subclasses of tuple. This means that C structures like those returned by :func:`os.stat`, - :func:`time.gmtime`, and :attr:`sys.version_info` now work like a + :func:`time.gmtime`, and :data:`sys.version_info` now work like a :term:`named tuple` and now work with functions and methods that expect a tuple as an argument. This is a big step forward in making the C structures as flexible as their pure Python counterparts: @@ -598,7 +598,7 @@ Some smaller changes made to the core Python language are: module, or on the command line. A :exc:`ResourceWarning` is issued at interpreter shutdown if the - :data:`gc.garbage` list isn't empty, and if :attr:`gc.DEBUG_UNCOLLECTABLE` is + :data:`gc.garbage` list isn't empty, and if :const:`gc.DEBUG_UNCOLLECTABLE` is set, all uncollectable objects are printed. This is meant to make the programmer aware that their code contains object finalization issues. @@ -623,7 +623,7 @@ Some smaller changes made to the core Python language are: :class:`collections.Sequence` :term:`abstract base class`. As a result, the language will have a more uniform API. In addition, :class:`range` objects now support slicing and negative indices, even with values larger than - :attr:`sys.maxsize`. This makes *range* more interoperable with lists:: + :data:`sys.maxsize`. This makes *range* more interoperable with lists:: >>> range(0, 100, 2).count(10) 1 @@ -662,7 +662,7 @@ Python's standard library has undergone significant maintenance efforts and quality improvements. The biggest news for Python 3.2 is that the :mod:`email` package, :mod:`mailbox` -module, and :mod:`nntplib` modules now work correctly with the bytes/text model +module, and :mod:`!nntplib` modules now work correctly with the bytes/text model in Python 3. For the first time, there is correct handling of messages with mixed encodings. @@ -1007,13 +1007,13 @@ datetime and time after 1900. The new supported year range is from 1000 to 9999 inclusive. * Whenever a two-digit year is used in a time tuple, the interpretation has been - governed by :attr:`time.accept2dyear`. The default is ``True`` which means that + governed by :data:`time.accept2dyear`. The default is ``True`` which means that for a two-digit year, the century is guessed according to the POSIX rules governing the ``%y`` strptime format. Starting with Py3.2, use of the century guessing heuristic will emit a :exc:`DeprecationWarning`. Instead, it is recommended that - :attr:`time.accept2dyear` be set to ``False`` so that large date ranges + :data:`time.accept2dyear` be set to ``False`` so that large date ranges can be used without guesswork:: >>> import time, warnings @@ -1031,7 +1031,7 @@ datetime and time 'Fri Jan 1 12:34:56 11' Several functions now have significantly expanded date ranges. When - :attr:`time.accept2dyear` is false, the :func:`time.asctime` function will + :data:`time.accept2dyear` is false, the :func:`time.asctime` function will accept any year that fits in a C int, while the :func:`time.mktime` and :func:`time.strftime` functions will accept the full range supported by the corresponding operating system functions. @@ -1194,11 +1194,11 @@ can be set to "$" for the shell-style formatting provided by If no configuration is set-up before a logging event occurs, there is now a default configuration using a :class:`~logging.StreamHandler` directed to -:attr:`sys.stderr` for events of ``WARNING`` level or higher. Formerly, an +:data:`sys.stderr` for events of ``WARNING`` level or higher. Formerly, an event occurring before a configuration was set-up would either raise an exception or silently drop the event depending on the value of -:attr:`logging.raiseExceptions`. The new default handler is stored in -:attr:`logging.lastResort`. +:data:`logging.raiseExceptions`. The new default handler is stored in +:data:`logging.lastResort`. The use of filters has been simplified. Instead of creating a :class:`~logging.Filter` object, the predicate can be any Python callable that @@ -1300,7 +1300,7 @@ values are equal (:issue:`8188`):: hash(Decimal("1.5")) == hash(complex(1.5, 0)) Some of the hashing details are exposed through a new attribute, -:attr:`sys.hash_info`, which describes the bit width of the hash value, the +:data:`sys.hash_info`, which describes the bit width of the hash value, the prime modulus, the hash values for *infinity* and *nan*, and the multiplier used for the imaginary part of a number: @@ -1388,7 +1388,7 @@ select ------ The :mod:`select` module now exposes a new, constant attribute, -:attr:`~select.PIPE_BUF`, which gives the minimum number of bytes which are +:const:`~select.PIPE_BUF`, which gives the minimum number of bytes which are guaranteed not to block when :func:`select.select` says a pipe is ready for writing. @@ -1529,7 +1529,7 @@ filenames: b'Sehensw\xc3\xbcrdigkeiten' Some operating systems allow direct access to encoded bytes in the -environment. If so, the :attr:`os.supports_bytes_environ` constant will be +environment. If so, the :const:`os.supports_bytes_environ` constant will be true. For direct access to encoded environment variables (if available), @@ -1666,9 +1666,9 @@ for secure (encrypted, authenticated) internet connections: algorithm" error. * The version of OpenSSL being used is now accessible using the module - attributes :data:`ssl.OPENSSL_VERSION` (a string), - :data:`ssl.OPENSSL_VERSION_INFO` (a 5-tuple), and - :data:`ssl.OPENSSL_VERSION_NUMBER` (an integer). + attributes :const:`ssl.OPENSSL_VERSION` (a string), + :const:`ssl.OPENSSL_VERSION_INFO` (a 5-tuple), and + :const:`ssl.OPENSSL_VERSION_NUMBER` (an integer). (Contributed by Antoine Pitrou in :issue:`8850`, :issue:`1589`, :issue:`8322`, :issue:`5639`, :issue:`4870`, :issue:`8484`, and :issue:`8321`.) @@ -1676,13 +1676,13 @@ for secure (encrypted, authenticated) internet connections: nntp ---- -The :mod:`nntplib` module has a revamped implementation with better bytes and +The :mod:`!nntplib` module has a revamped implementation with better bytes and text semantics as well as more practical APIs. These improvements break compatibility with the nntplib version in Python 3.1, which was partly dysfunctional in itself. Support for secure connections through both implicit (using -:class:`nntplib.NNTP_SSL`) and explicit (using :meth:`nntplib.NNTP.starttls`) +:class:`!nntplib.NNTP_SSL`) and explicit (using :meth:`!nntplib.NNTP.starttls`) TLS has also been added. (Contributed by Antoine Pitrou in :issue:`9360` and Andrew Vant in :issue:`1926`.) @@ -2302,7 +2302,7 @@ turtledemo The demonstration code for the :mod:`turtle` module was moved from the *Demo* directory to main library. It includes over a dozen sample scripts with -lively displays. Being on :attr:`sys.path`, it can now be run directly +lively displays. Being on :data:`sys.path`, it can now be run directly from the command-line: .. code-block:: shell-session @@ -2566,10 +2566,10 @@ Changes to Python's build process and to the C API include: (:issue:`2443`). * A new C API function :c:func:`PySys_SetArgvEx` allows an embedded interpreter - to set :attr:`sys.argv` without also modifying :attr:`sys.path` + to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). -* :c:macro:`PyEval_CallObject` is now only available in macro form. The +* :c:func:`!PyEval_CallObject` is now only available in macro form. The function declaration, which was kept for backwards compatibility reasons, is now removed -- the macro was introduced in 1997 (:issue:`8276`). @@ -2643,7 +2643,7 @@ require changes to your code: * ``""`` is now a valid value and is no longer automatically converted to an empty string. For empty strings, use ``"option ="`` in a line. -* The :mod:`nntplib` module was reworked extensively, meaning that its APIs +* The :mod:`!nntplib` module was reworked extensively, meaning that its APIs are often incompatible with the 3.1 APIs. * :class:`bytearray` objects can no longer be used as filenames; instead, @@ -2658,7 +2658,7 @@ require changes to your code: * "t#" format has been removed: use "s#" or "s*" instead * "w" and "w#" formats has been removed: use "w*" instead -* The :c:type:`PyCObject` type, deprecated in 3.1, has been removed. To wrap +* The :c:type:`!PyCObject` type, deprecated in 3.1, has been removed. To wrap opaque C pointers in Python objects, the :c:type:`PyCapsule` API should be used instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. @@ -2731,15 +2731,15 @@ require changes to your code: (Contributed by Antoine Pitrou, :issue:`10272`.) -* The misleading functions :c:func:`PyEval_AcquireLock()` and - :c:func:`PyEval_ReleaseLock()` have been officially deprecated. The - thread-state aware APIs (such as :c:func:`PyEval_SaveThread()` - and :c:func:`PyEval_RestoreThread()`) should be used instead. +* The misleading functions :c:func:`!PyEval_AcquireLock` and + :c:func:`!PyEval_ReleaseLock` have been officially deprecated. The + thread-state aware APIs (such as :c:func:`PyEval_SaveThread` + and :c:func:`PyEval_RestoreThread`) should be used instead. * Due to security risks, :func:`asyncore.handle_accept` has been deprecated, and a new function, :func:`asyncore.handle_accepted`, was added to replace it. (Contributed by Giampaolo Rodola in :issue:`6706`.) -* Due to the new :term:`GIL` implementation, :c:func:`PyEval_InitThreads()` - cannot be called before :c:func:`Py_Initialize()` anymore. +* Due to the new :term:`GIL` implementation, :c:func:`!PyEval_InitThreads` + cannot be called before :c:func:`Py_Initialize` anymore. diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index f121652ba51cbc..5674bc7f359b72 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -249,7 +249,7 @@ Changes introduced by :pep:`393` are the following: non-BMP code points. * The value of :data:`sys.maxunicode` is now always ``1114111`` (``0x10FFFF`` - in hexadecimal). The :c:func:`PyUnicode_GetMax` function still returns + in hexadecimal). The :c:func:`!PyUnicode_GetMax` function still returns either ``0xFFFF`` or ``0x10FFFF`` for backward compatibility, and it should not be used with the new Unicode API (see :issue:`13054`). @@ -648,7 +648,7 @@ PEP 421: Adding sys.implementation A new attribute on the :mod:`sys` module exposes details specific to the implementation of the currently running interpreter. The initial set of -attributes on :attr:`sys.implementation` are ``name``, ``version``, +attributes on :data:`sys.implementation` are ``name``, ``version``, ``hexversion``, and ``cache_tag``. The intention of ``sys.implementation`` is to consolidate into one namespace @@ -719,7 +719,7 @@ and does not enforce any method requirements. In terms of finders, :class:`importlib.machinery.FileFinder` exposes the mechanism used to search for source and bytecode files of a module. Previously -this class was an implicit member of :attr:`sys.path_hooks`. +this class was an implicit member of :data:`sys.path_hooks`. For loaders, the new abstract base class :class:`importlib.abc.FileLoader` helps write a loader that uses the file system as the storage mechanism for a module's @@ -735,7 +735,7 @@ provide the full name of the module now instead of just the tail end of the module's name. The :func:`importlib.invalidate_caches` function will now call the method with -the same name on all finders cached in :attr:`sys.path_importer_cache` to help +the same name on all finders cached in :data:`sys.path_importer_cache` to help clean up any stored state as necessary. Visible Changes @@ -745,8 +745,8 @@ For potential required changes to code, see the `Porting Python code`_ section. Beyond the expanse of what :mod:`importlib` now exposes, there are other -visible changes to import. The biggest is that :attr:`sys.meta_path` and -:attr:`sys.path_hooks` now store all of the meta path finders and path entry +visible changes to import. The biggest is that :data:`sys.meta_path` and +:data:`sys.path_hooks` now store all of the meta path finders and path entry hooks used by import. Previously the finders were implicit and hidden within the C code of import instead of being directly exposed. This means that one can now easily remove or change the order of the various finders to fit one's needs. @@ -761,9 +761,9 @@ Loaders are also now expected to set the ``__package__`` attribute from :pep:`366`. Once again, import itself is already setting this on all loaders from :mod:`importlib` and import itself is setting the attribute post-load. -``None`` is now inserted into :attr:`sys.path_importer_cache` when no finder -can be found on :attr:`sys.path_hooks`. Since :class:`!imp.NullImporter` is not -directly exposed on :attr:`sys.path_hooks` it could no longer be relied upon to +``None`` is now inserted into :data:`sys.path_importer_cache` when no finder +can be found on :data:`sys.path_hooks`. Since :class:`!imp.NullImporter` is not +directly exposed on :data:`sys.path_hooks` it could no longer be relied upon to always be available to use as a value representing no finder found. All other changes relate to semantic changes which should be taken into @@ -842,7 +842,7 @@ Builtin functions and types * :func:`open` gets a new *opener* parameter: the underlying file descriptor for the file object is then obtained by calling *opener* with (*file*, - *flags*). It can be used to use custom flags like :data:`os.O_CLOEXEC` for + *flags*). It can be used to use custom flags like :const:`os.O_CLOEXEC` for example. The ``'x'`` mode was added: open for exclusive creation, failing if the file already exists. * :func:`print`: added the *flush* keyword argument. If the *flush* keyword @@ -917,12 +917,12 @@ abstract methods. The recommended approach to declaring abstract descriptors is now to provide :attr:`__isabstractmethod__` as a dynamically updated property. The built-in descriptors have been updated accordingly. - * :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. +* :class:`abc.abstractproperty` has been deprecated, use :class:`property` + with :func:`abc.abstractmethod` instead. +* :class:`abc.abstractclassmethod` has been deprecated, use + :class:`classmethod` with :func:`abc.abstractmethod` instead. +* :class:`abc.abstractstaticmethod` has been deprecated, use + :class:`staticmethod` with :func:`abc.abstractmethod` instead. (Contributed by Darren Dale in :issue:`11610`.) @@ -1052,40 +1052,40 @@ their ``__init__`` method (for example, file objects) or in their crypt ----- -Addition of salt and modular crypt format (hashing method) and the :func:`~crypt.mksalt` -function to the :mod:`crypt` module. +Addition of salt and modular crypt format (hashing method) and the :func:`~!crypt.mksalt` +function to the :mod:`!crypt` module. (:issue:`10924`) curses ------ - * If the :mod:`curses` module is linked to the ncursesw library, use Unicode - functions when Unicode strings or characters are passed (e.g. - :c:func:`waddwstr`), and bytes functions otherwise (e.g. :c:func:`waddstr`). - * Use the locale encoding instead of ``utf-8`` to encode Unicode strings. - * :class:`curses.window` has a new :attr:`curses.window.encoding` attribute. - * The :class:`curses.window` class has a new :meth:`~curses.window.get_wch` - method to get a wide character - * The :mod:`curses` module has a new :meth:`~curses.unget_wch` function to - push a wide character so the next :meth:`~curses.window.get_wch` will return - it +* If the :mod:`curses` module is linked to the ncursesw library, use Unicode + functions when Unicode strings or characters are passed (e.g. + :c:func:`waddwstr`), and bytes functions otherwise (e.g. :c:func:`waddstr`). +* Use the locale encoding instead of ``utf-8`` to encode Unicode strings. +* :class:`curses.window` has a new :attr:`curses.window.encoding` attribute. +* The :class:`curses.window` class has a new :meth:`~curses.window.get_wch` + method to get a wide character +* The :mod:`curses` module has a new :meth:`~curses.unget_wch` function to + push a wide character so the next :meth:`~curses.window.get_wch` will return + it (Contributed by Iñigo Serna in :issue:`6755`.) datetime -------- - * Equality comparisons between naive and aware :class:`~datetime.datetime` - instances now return :const:`False` instead of raising :exc:`TypeError` - (:issue:`15006`). - * New :meth:`datetime.datetime.timestamp` method: Return POSIX timestamp - corresponding to the :class:`~datetime.datetime` instance. - * The :meth:`datetime.datetime.strftime` method supports formatting years - older than 1000. - * The :meth:`datetime.datetime.astimezone` method can now be - called without arguments to convert datetime instance to the system - timezone. +* Equality comparisons between naive and aware :class:`~datetime.datetime` + instances now return :const:`False` instead of raising :exc:`TypeError` + (:issue:`15006`). +* New :meth:`datetime.datetime.timestamp` method: Return POSIX timestamp + corresponding to the :class:`~datetime.datetime` instance. +* The :meth:`datetime.datetime.strftime` method supports formatting years + older than 1000. +* The :meth:`datetime.datetime.astimezone` method can now be + called without arguments to convert datetime instance to the system + timezone. .. _new-decimal: @@ -1127,7 +1127,7 @@ Features * If Python is compiled without threads, the C version automatically disables the expensive thread local context machinery. In this case, - the variable :data:`~decimal.HAVE_THREADS` is set to ``False``. + the variable :const:`~decimal.HAVE_THREADS` is set to ``False``. API changes ~~~~~~~~~~~ @@ -1135,20 +1135,20 @@ API changes * The C module has the following context limits, depending on the machine architecture: - +-------------------+---------------------+------------------------------+ - | | 32-bit | 64-bit | - +===================+=====================+==============================+ - | :const:`MAX_PREC` | :const:`425000000` | :const:`999999999999999999` | - +-------------------+---------------------+------------------------------+ - | :const:`MAX_EMAX` | :const:`425000000` | :const:`999999999999999999` | - +-------------------+---------------------+------------------------------+ - | :const:`MIN_EMIN` | :const:`-425000000` | :const:`-999999999999999999` | - +-------------------+---------------------+------------------------------+ + +-------------------+----------------+-------------------------+ + | | 32-bit | 64-bit | + +===================+================+=========================+ + | :const:`MAX_PREC` | ``425000000`` | ``999999999999999999`` | + +-------------------+----------------+-------------------------+ + | :const:`MAX_EMAX` | ``425000000`` | ``999999999999999999`` | + +-------------------+----------------+-------------------------+ + | :const:`MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | + +-------------------+----------------+-------------------------+ * In the context templates (:class:`~decimal.DefaultContext`, :class:`~decimal.BasicContext` and :class:`~decimal.ExtendedContext`) the magnitude of :attr:`~decimal.Context.Emax` and - :attr:`~decimal.Context.Emin` has changed to :const:`999999`. + :attr:`~decimal.Context.Emin` has changed to ``999999``. * The :class:`~decimal.Decimal` constructor in decimal.py does not observe the context limits and converts values with arbitrary exponents or precision @@ -1210,25 +1210,25 @@ the ``Message`` object it is serializing. The default policy is The minimum set of controls implemented by all ``policy`` objects are: - .. tabularcolumns:: |l|L| +.. tabularcolumns:: |l|L| - =============== ======================================================= - max_line_length The maximum length, excluding the linesep character(s), - individual lines may have when a ``Message`` is - serialized. Defaults to 78. +=============== ======================================================= +max_line_length The maximum length, excluding the linesep character(s), + individual lines may have when a ``Message`` is + serialized. Defaults to 78. - linesep The character used to separate individual lines when a - ``Message`` is serialized. Defaults to ``\n``. +linesep The character used to separate individual lines when a + ``Message`` is serialized. Defaults to ``\n``. - cte_type ``7bit`` or ``8bit``. ``8bit`` applies only to a - ``Bytes`` ``generator``, and means that non-ASCII may - be used where allowed by the protocol (or where it - exists in the original input). +cte_type ``7bit`` or ``8bit``. ``8bit`` applies only to a + ``Bytes`` ``generator``, and means that non-ASCII may + be used where allowed by the protocol (or where it + exists in the original input). - raise_on_defect Causes a ``parser`` to raise error when defects are - encountered instead of adding them to the ``Message`` - object's ``defects`` list. - =============== ======================================================= +raise_on_defect Causes a ``parser`` to raise error when defects are + encountered instead of adding them to the ``Message`` + object's ``defects`` list. +=============== ======================================================= A new policy instance, with new settings, is created using the :meth:`~email.policy.Policy.clone` method of policy objects. ``clone`` takes @@ -1263,21 +1263,21 @@ removal of the code) may occur if deemed necessary by the core developers. The new policies are instances of :class:`~email.policy.EmailPolicy`, and add the following additional controls: - .. tabularcolumns:: |l|L| +.. tabularcolumns:: |l|L| - =============== ======================================================= - refold_source Controls whether or not headers parsed by a - :mod:`~email.parser` are refolded by the - :mod:`~email.generator`. It can be ``none``, ``long``, - or ``all``. The default is ``long``, which means that - source headers with a line longer than - ``max_line_length`` get refolded. ``none`` means no - line get refolded, and ``all`` means that all lines - get refolded. +=============== ======================================================= +refold_source Controls whether or not headers parsed by a + :mod:`~email.parser` are refolded by the + :mod:`~email.generator`. It can be ``none``, ``long``, + or ``all``. The default is ``long``, which means that + source headers with a line longer than + ``max_line_length`` get refolded. ``none`` means no + line get refolded, and ``all`` means that all lines + get refolded. - header_factory A callable that take a ``name`` and ``value`` and - produces a custom header object. - =============== ======================================================= +header_factory A callable that take a ``name`` and ``value`` and + produces a custom header object. +=============== ======================================================= The ``header_factory`` is the key to the new features provided by the new policies. When one of the new policies is used, any header retrieved from @@ -1352,18 +1352,18 @@ API. New utility functions: - * :func:`~email.utils.format_datetime`: given a :class:`~datetime.datetime`, - produce a string formatted for use in an email header. +* :func:`~email.utils.format_datetime`: given a :class:`~datetime.datetime`, + produce a string formatted for use in an email header. - * :func:`~email.utils.parsedate_to_datetime`: given a date string from - an email header, convert it into an aware :class:`~datetime.datetime`, - or a naive :class:`~datetime.datetime` if the offset is ``-0000``. +* :func:`~email.utils.parsedate_to_datetime`: given a date string from + an email header, convert it into an aware :class:`~datetime.datetime`, + or a naive :class:`~datetime.datetime` if the offset is ``-0000``. - * :func:`~email.utils.localtime`: With no argument, returns the - current local time as an aware :class:`~datetime.datetime` using the local - :class:`~datetime.timezone`. Given an aware :class:`~datetime.datetime`, - converts it into an aware :class:`~datetime.datetime` using the - local :class:`~datetime.timezone`. +* :func:`~email.utils.localtime`: With no argument, returns the + current local time as an aware :class:`~datetime.datetime` using the local + :class:`~datetime.timezone`. Given an aware :class:`~datetime.datetime`, + converts it into an aware :class:`~datetime.datetime` using the + local :class:`~datetime.timezone`. ftplib @@ -1558,7 +1558,7 @@ Schlawack in :issue:`12708`.) nntplib ------- -The :class:`nntplib.NNTP` class now supports the context management protocol to +The :class:`!nntplib.NNTP` class now supports the context management protocol to unconditionally consume :exc:`socket.error` exceptions and to close the NNTP connection when done:: @@ -1576,8 +1576,8 @@ os -- * The :mod:`os` module has a new :func:`~os.pipe2` function that makes it - possible to create a pipe with :data:`~os.O_CLOEXEC` or - :data:`~os.O_NONBLOCK` flags set atomically. This is especially useful to + possible to create a pipe with :const:`~os.O_CLOEXEC` or + :const:`~os.O_NONBLOCK` flags set atomically. This is especially useful to avoid race conditions in multi-threaded programs. * The :mod:`os` module has a new :func:`~os.sendfile` function which provides @@ -1691,9 +1691,9 @@ os * Some platforms now support additional constants for the :func:`~os.lseek` function, such as ``os.SEEK_HOLE`` and ``os.SEEK_DATA``. -* New constants :data:`~os.RTLD_LAZY`, :data:`~os.RTLD_NOW`, - :data:`~os.RTLD_GLOBAL`, :data:`~os.RTLD_LOCAL`, :data:`~os.RTLD_NODELETE`, - :data:`~os.RTLD_NOLOAD`, and :data:`~os.RTLD_DEEPBIND` are available on +* New constants :const:`~os.RTLD_LAZY`, :const:`~os.RTLD_NOW`, + :const:`~os.RTLD_GLOBAL`, :const:`~os.RTLD_LOCAL`, :const:`~os.RTLD_NODELETE`, + :const:`~os.RTLD_NOLOAD`, and :const:`~os.RTLD_DEEPBIND` are available on platforms that support them. These are for use with the :func:`sys.setdlopenflags` function, and supersede the similar constants defined in :mod:`ctypes` and :mod:`DLFCN`. (Contributed by Victor Stinner @@ -1778,7 +1778,7 @@ shlex ----- The previously undocumented helper function ``quote`` from the -:mod:`pipes` modules has been moved to the :mod:`shlex` module and +:mod:`!pipes` modules has been moved to the :mod:`shlex` module and documented. :func:`~shlex.quote` properly escapes all characters in a string that might be otherwise given special meaning by the shell. @@ -1952,7 +1952,7 @@ ssl * You can query the SSL compression algorithm used by an SSL socket, thanks to its new :meth:`~ssl.SSLSocket.compression` method. The new attribute - :attr:`~ssl.OP_NO_COMPRESSION` can be used to disable compression. + :const:`~ssl.OP_NO_COMPRESSION` can be used to disable compression. (Contributed by Antoine Pitrou in :issue:`13634`.) * Support has been added for the Next Protocol Negotiation extension using @@ -1966,7 +1966,7 @@ ssl * The :func:`~ssl.get_server_certificate` function now supports IPv6. (Contributed by Charles-François Natali in :issue:`11811`.) -* New attribute :attr:`~ssl.OP_CIPHER_SERVER_PREFERENCE` allows setting +* New attribute :const:`~ssl.OP_CIPHER_SERVER_PREFERENCE` allows setting SSLv3 server sockets to use the server's cipher ordering preference rather than the client's (:issue:`13635`). @@ -1984,7 +1984,7 @@ the form '-rwxrwxrwx'. struct ------ -The :mod:`struct` module now supports ``ssize_t`` and ``size_t`` via the +The :mod:`struct` module now supports :c:type:`ssize_t` and :c:type:`size_t` via the new codes ``n`` and ``N``, respectively. (Contributed by Antoine Pitrou in :issue:`3163`.) @@ -1995,7 +1995,7 @@ subprocess Command strings can now be bytes objects on posix platforms. (Contributed by Victor Stinner in :issue:`8513`.) -A new constant :data:`~subprocess.DEVNULL` allows suppressing output in a +A new constant :const:`~subprocess.DEVNULL` allows suppressing output in a platform-independent fashion. (Contributed by Ross Lagerwall in :issue:`5870`.) @@ -2067,7 +2067,7 @@ The :pep:`418` added new functions to the :mod:`time` module: Other new functions: * :func:`~time.clock_getres`, :func:`~time.clock_gettime` and - :func:`~time.clock_settime` functions with ``CLOCK_xxx`` constants. + :func:`~time.clock_settime` functions with :samp:`CLOCK_{xxx}` constants. (Contributed by Victor Stinner in :issue:`10278`.) To improve cross platform consistency, :func:`~time.sleep` now raises a @@ -2141,7 +2141,7 @@ New attribute :attr:`zlib.Decompress.eof` makes it possible to distinguish between a properly formed compressed stream and an incomplete or truncated one. (Contributed by Nadeem Vawda in :issue:`12646`.) -New attribute :attr:`zlib.ZLIB_RUNTIME_VERSION` reports the version string of +New attribute :const:`zlib.ZLIB_RUNTIME_VERSION` reports the version string of the underlying ``zlib`` library that is loaded at runtime. (Contributed by Torsten Landschoff in :issue:`12306`.) @@ -2195,8 +2195,8 @@ Changes to Python's build process and to the C API include: * :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsUCS4Copy` * :c:macro:`PyUnicode_DATA`, :c:macro:`PyUnicode_1BYTE_DATA`, :c:macro:`PyUnicode_2BYTE_DATA`, :c:macro:`PyUnicode_4BYTE_DATA` - * :c:macro:`PyUnicode_KIND` with :c:type:`PyUnicode_Kind` enum: - :c:data:`PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, + * :c:macro:`PyUnicode_KIND` with :c:enum:`PyUnicode_Kind` enum: + :c:data:`!PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, :c:data:`PyUnicode_2BYTE_KIND`, :c:data:`PyUnicode_4BYTE_KIND` * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` * :c:macro:`PyUnicode_MAX_CHAR_VALUE` @@ -2270,58 +2270,58 @@ removed in Python 4. All functions using this type are deprecated: Unicode functions and methods using :c:type:`Py_UNICODE` and :c:expr:`Py_UNICODE*` types: -* :c:macro:`PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or +* :c:macro:`!PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_FromKindAndData` -* :c:macro:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_AsUnicode`, - :c:func:`PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` -* :c:macro:`PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with +* :c:macro:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_AsUnicode`, + :c:func:`!PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` +* :c:macro:`!PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with :c:macro:`PyUnicode_READ` and :c:macro:`PyUnicode_WRITE` -* :c:macro:`PyUnicode_GET_SIZE`, :c:func:`PyUnicode_GetSize`: use +* :c:macro:`!PyUnicode_GET_SIZE`, :c:func:`!PyUnicode_GetSize`: use :c:macro:`PyUnicode_GET_LENGTH` or :c:func:`PyUnicode_GetLength` -* :c:macro:`PyUnicode_GET_DATA_SIZE`: use +* :c:macro:`!PyUnicode_GET_DATA_SIZE`: use ``PyUnicode_GET_LENGTH(str) * PyUnicode_KIND(str)`` (only work on ready strings) -* :c:func:`PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or +* :c:func:`!PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or :c:func:`PyUnicode_AsWideCharString` -* :c:func:`PyUnicode_GetMax` +* :c:func:`!PyUnicode_GetMax` Functions and macros manipulating Py_UNICODE* strings: -* :c:macro:`Py_UNICODE_strlen`: use :c:func:`PyUnicode_GetLength` or +* :c:macro:`!Py_UNICODE_strlen()`: use :c:func:`PyUnicode_GetLength` or :c:macro:`PyUnicode_GET_LENGTH` -* :c:macro:`Py_UNICODE_strcat`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcat()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_FromFormat` -* :c:macro:`Py_UNICODE_strcpy`, :c:macro:`Py_UNICODE_strncpy`, - :c:macro:`Py_UNICODE_COPY`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcpy()`, :c:macro:`!Py_UNICODE_strncpy()`, + :c:macro:`!Py_UNICODE_COPY()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` -* :c:macro:`Py_UNICODE_strcmp`: use :c:func:`PyUnicode_Compare` -* :c:macro:`Py_UNICODE_strncmp`: use :c:func:`PyUnicode_Tailmatch` -* :c:macro:`Py_UNICODE_strchr`, :c:macro:`Py_UNICODE_strrchr`: use +* :c:macro:`!Py_UNICODE_strcmp()`: use :c:func:`PyUnicode_Compare` +* :c:macro:`!Py_UNICODE_strncmp()`: use :c:func:`PyUnicode_Tailmatch` +* :c:macro:`!Py_UNICODE_strchr()`, :c:macro:`!Py_UNICODE_strrchr()`: use :c:func:`PyUnicode_FindChar` -* :c:macro:`Py_UNICODE_FILL`: use :c:func:`PyUnicode_Fill` -* :c:macro:`Py_UNICODE_MATCH` +* :c:macro:`!Py_UNICODE_FILL()`: use :c:func:`PyUnicode_Fill` +* :c:macro:`!Py_UNICODE_MATCH` Encoders: -* :c:func:`PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` -* :c:func:`PyUnicode_EncodeUTF7` -* :c:func:`PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or +* :c:func:`!PyUnicode_Encode`: use :c:func:`!PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_EncodeUTF7` +* :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` -* :c:func:`PyUnicode_EncodeUTF32` -* :c:func:`PyUnicode_EncodeUTF16` -* :c:func:`PyUnicode_EncodeUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeUTF32` +* :c:func:`!PyUnicode_EncodeUTF16` +* :c:func:`!PyUnicode_EncodeUnicodeEscape` use :c:func:`PyUnicode_AsUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeRawUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeRawUnicodeEscape` use :c:func:`PyUnicode_AsRawUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` -* :c:func:`PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` -* :c:func:`PyUnicode_EncodeCharmap` -* :c:func:`PyUnicode_TranslateCharmap` -* :c:func:`PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or +* :c:func:`!PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` +* :c:func:`!PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` +* :c:func:`!PyUnicode_EncodeCharmap` +* :c:func:`!PyUnicode_TranslateCharmap` +* :c:func:`!PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or :c:func:`PyUnicode_EncodeCodePage` (with ``CP_ACP`` code_page) -* :c:func:`PyUnicode_EncodeDecimal`, - :c:func:`PyUnicode_TransformDecimalToASCII` +* :c:func:`!PyUnicode_EncodeDecimal`, + :c:func:`!PyUnicode_TransformDecimalToASCII` Deprecated features @@ -2378,16 +2378,16 @@ Porting Python code * :func:`__import__` no longer allows one to use an index value other than 0 for top-level modules. E.g. ``__import__('sys', level=1)`` is now an error. -* Because :attr:`sys.meta_path` and :attr:`sys.path_hooks` now have finders on +* Because :data:`sys.meta_path` and :data:`sys.path_hooks` now have finders on them by default, you will most likely want to use :meth:`list.insert` instead of :meth:`list.append` to add to those lists. -* Because ``None`` is now inserted into :attr:`sys.path_importer_cache`, if you +* Because ``None`` is now inserted into :data:`sys.path_importer_cache`, if you are clearing out entries in the dictionary of paths that do not have a finder, you will need to remove keys paired with values of ``None`` **and** :class:`!imp.NullImporter` to be backwards-compatible. This will lead to extra overhead on older versions of Python that re-insert ``None`` into - :attr:`sys.path_importer_cache` where it represents the use of implicit + :data:`sys.path_importer_cache` where it represents the use of implicit finders, but semantically it should not change anything. * :class:`!importlib.abc.Finder` no longer specifies a ``find_module()`` abstract @@ -2445,7 +2445,7 @@ Porting Python code error instead of sleeping forever. It has always raised an error on posix. * The ``ast.__version__`` constant has been removed. If you need to - make decisions affected by the AST version, use :attr:`sys.version_info` + make decisions affected by the AST version, use :data:`sys.version_info` to make the decision. * Code that used to work around the fact that the :mod:`threading` module used @@ -2462,7 +2462,7 @@ Porting C code -------------- * In the course of changes to the buffer API the undocumented - :c:member:`~Py_buffer.smalltable` member of the + :c:member:`!smalltable` member of the :c:type:`Py_buffer` structure has been removed and the layout of the :c:type:`PyMemoryViewObject` has changed. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 45bb91833a352b..2ddab76814369e 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -605,15 +605,15 @@ Using ``ABC`` as a base class has essentially the same effect as specifying aifc ---- -The :meth:`~aifc.aifc.getparams` method now returns a namedtuple rather than a +The :meth:`~!aifc.aifc.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.) -:func:`aifc.open` now supports the context management protocol: when used in a -:keyword:`with` block, the :meth:`~aifc.aifc.close` method of the returned +:func:`!aifc.open` now supports the context management protocol: when used in a +:keyword:`with` block, the :meth:`~!aifc.aifc.close` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchacha in :issue:`16486`.) -The :meth:`~aifc.aifc.writeframesraw` and :meth:`~aifc.aifc.writeframes` +The :meth:`~!aifc.aifc.writeframesraw` and :meth:`~!aifc.aifc.writeframes` methods now accept any :term:`bytes-like object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) @@ -629,14 +629,14 @@ by Lucas Maystre in :issue:`11175`.) audioop ------- -:mod:`audioop` now supports 24-bit samples. (Contributed by Serhiy Storchaka +:mod:`!audioop` now supports 24-bit samples. (Contributed by Serhiy Storchaka in :issue:`12866`.) -New :func:`~audioop.byteswap` function converts big-endian samples to +New :func:`~!audioop.byteswap` function converts big-endian samples to little-endian and vice versa. (Contributed by Serhiy Storchaka in :issue:`19641`.) -All :mod:`audioop` functions now accept any :term:`bytes-like object`. Strings +All :mod:`!audioop` functions now accept any :term:`bytes-like object`. Strings are not accepted: they didn't work before, now they raise an error right away. (Contributed by Serhiy Storchaka in :issue:`16685`.) @@ -775,7 +775,7 @@ of a given opcode and argument, information that is not otherwise available. doctest ------- -A new :ref:`option flag `, :data:`~doctest.FAIL_FAST`, halts +A new :ref:`option flag `, :const:`~doctest.FAIL_FAST`, halts test running as soon as the first failure is detected. (Contributed by R. David Murray and Daniel Urban in :issue:`16522`.) @@ -841,7 +841,7 @@ for example, if the file might have been changed and re-checked in less time than the resolution of a particular filesystem's file modification time field. (Contributed by Mark Levitt in :issue:`18149`.) -New module attribute :data:`~filecmp.DEFAULT_IGNORES` provides the list of +New module attribute :const:`~filecmp.DEFAULT_IGNORES` provides the list of directories that are used as the default value for the *ignore* parameter of the :func:`~filecmp.dircmp` function. (Contributed by Eli Bendersky in :issue:`15442`.) @@ -1189,7 +1189,7 @@ Windows). (Contributed by Brian Curtin in :issue:`11939`.) root on Windows. (Contributed by Tim Golden in :issue:`9035`.) :func:`os.open` supports two new flags on platforms that provide them, -:data:`~os.O_PATH` (un-opened file descriptor), and :data:`~os.O_TMPFILE` +:const:`~os.O_PATH` (un-opened file descriptor), and :const:`~os.O_TMPFILE` (unnamed temporary file; as of 3.4.0 release available only on Linux systems with a kernel version of 3.11 or newer that have uapi headers). (Contributed by Christian Heimes in :issue:`18673` and Benjamin Peterson, respectively.) @@ -1238,8 +1238,8 @@ plistlib stdlib serialization protocols, with new :func:`~plistlib.load`, :func:`~plistlib.dump`, :func:`~plistlib.loads`, and :func:`~plistlib.dumps` functions. (The older API is now deprecated.) In addition to the already -supported XML plist format (:data:`~plistlib.FMT_XML`), it also now supports -the binary plist format (:data:`~plistlib.FMT_BINARY`). (Contributed by Ronald +supported XML plist format (:const:`~plistlib.FMT_XML`), it also now supports +the binary plist format (:const:`~plistlib.FMT_BINARY`). (Contributed by Ronald Oussoren and others in :issue:`14455`.) @@ -1323,14 +1323,14 @@ ability to query or set the resource limits for processes other than the one making the call. (Contributed by Christian Heimes in :issue:`16595`.) On Linux kernel version 2.6.36 or later, there are also some new -Linux specific constants: :attr:`~resource.RLIMIT_MSGQUEUE`, -:attr:`~resource.RLIMIT_NICE`, :attr:`~resource.RLIMIT_RTPRIO`, -:attr:`~resource.RLIMIT_RTTIME`, and :attr:`~resource.RLIMIT_SIGPENDING`. +Linux specific constants: :const:`~resource.RLIMIT_MSGQUEUE`, +:const:`~resource.RLIMIT_NICE`, :const:`~resource.RLIMIT_RTPRIO`, +:const:`~resource.RLIMIT_RTTIME`, and :const:`~resource.RLIMIT_SIGPENDING`. (Contributed by Christian Heimes in :issue:`19324`.) On FreeBSD version 9 and later, there some new FreeBSD specific constants: -:attr:`~resource.RLIMIT_SBSIZE`, :attr:`~resource.RLIMIT_SWAP`, and -:attr:`~resource.RLIMIT_NPTS`. (Contributed by Claudiu Popa in +:const:`~resource.RLIMIT_SBSIZE`, :const:`~resource.RLIMIT_SWAP`, and +:const:`~resource.RLIMIT_NPTS`. (Contributed by Claudiu Popa in :issue:`19343`.) @@ -1388,7 +1388,7 @@ try/except statement by code that only cares whether or not an error occurred. socket ------ -The socket module now supports the :data:`~socket.CAN_BCM` protocol on +The socket module now supports the :const:`~socket.CAN_BCM` protocol on platforms that support it. (Contributed by Brian Thorne in :issue:`15359`.) Socket objects have new methods to get or set their :ref:`inheritable flag @@ -1399,7 +1399,7 @@ The ``socket.AF_*`` and ``socket.SOCK_*`` constants are now enumeration values using the new :mod:`enum` module. This allows meaningful names to be printed during debugging, instead of integer "magic numbers". -The :data:`~socket.AF_LINK` constant is now available on BSD and OSX. +The :const:`~socket.AF_LINK` constant is now available on BSD and OSX. :func:`~socket.inet_pton` and :func:`~socket.inet_ntop` are now supported on Windows. (Contributed by Atsuo Ishimoto in :issue:`7171`.) @@ -1460,8 +1460,8 @@ Heimes in :issue:`18147`.) If OpenSSL 0.9.8 or later is available, :class:`~ssl.SSLContext` has a new attribute :attr:`~ssl.SSLContext.verify_flags` that can be used to control the certificate verification process by setting it to some combination of the new -constants :data:`~ssl.VERIFY_DEFAULT`, :data:`~ssl.VERIFY_CRL_CHECK_LEAF`, -:data:`~ssl.VERIFY_CRL_CHECK_CHAIN`, or :data:`~ssl.VERIFY_X509_STRICT`. +constants :const:`~ssl.VERIFY_DEFAULT`, :const:`~ssl.VERIFY_CRL_CHECK_LEAF`, +:const:`~ssl.VERIFY_CRL_CHECK_CHAIN`, or :const:`~ssl.VERIFY_X509_STRICT`. OpenSSL does not do any CRL verification by default. (Contributed by Christien Heimes in :issue:`8813`.) @@ -1500,7 +1500,7 @@ implementation is required as most of the values aren't standardized and are platform-dependent. (Contributed by Christian Heimes in :issue:`11016`.) The module supports new :mod:`~stat.ST_MODE` flags, :mod:`~stat.S_IFDOOR`, -:attr:`~stat.S_IFPORT`, and :attr:`~stat.S_IFWHT`. (Contributed by +:const:`~stat.S_IFPORT`, and :const:`~stat.S_IFWHT`. (Contributed by Christian Hiemes in :issue:`11016`.) @@ -1528,10 +1528,10 @@ work on Windows. This change was actually inadvertently made in 3.3.4. sunau ----- -The :meth:`~sunau.getparams` method now returns a namedtuple rather than a +The :meth:`~!sunau.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) -:meth:`sunau.open` now supports the context management protocol: when used in a +:meth:`!sunau.open` now supports the context management protocol: when used in a :keyword:`with` block, the ``close`` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchaka in :issue:`18878`.) @@ -1540,8 +1540,8 @@ in :issue:`18878`.) support for writing 24 sample using the module. (Contributed by Serhiy Storchaka in :issue:`19261`.) -The :meth:`~sunau.AU_write.writeframesraw` and -:meth:`~sunau.AU_write.writeframes` methods now accept any :term:`bytes-like +The :meth:`~!sunau.AU_write.writeframesraw` and +:meth:`~!sunau.AU_write.writeframes` methods now accept any :term:`bytes-like object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) @@ -1849,7 +1849,7 @@ Python's default implementation to a SipHash implementation on platforms that have a 64 bit data type. Any performance differences in comparison with the older FNV algorithm are trivial. -The PEP adds additional fields to the :attr:`sys.hash_info` named tuple to +The PEP adds additional fields to the :data:`sys.hash_info` named tuple to describe the hash algorithm in use by the currently executing binary. Otherwise, the PEP does not alter any existing CPython APIs. @@ -1891,7 +1891,7 @@ Other Build and C API Changes allowing retrieval of function pointers from named type slots when using the limited API. (Contributed by Martin von Löwis in :issue:`17162`.) -* The new :c:func:`Py_SetStandardStreamEncoding` pre-initialization API +* The new :c:func:`!Py_SetStandardStreamEncoding` pre-initialization API allows applications embedding the CPython interpreter to reliably force a particular encoding and error handler for the standard streams. (Contributed by Bastien Montagne and Nick Coghlan in :issue:`16129`.) @@ -2085,7 +2085,7 @@ Deprecations in the Python API :meth:`importlib.abc.MetaPathFinder.find_spec`; :meth:`!importlib.abc.PathEntryFinder.find_loader` and :meth:`!find_module` are replaced by - :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the ``xxxLoader`` ABC + :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the :samp:`{xxx}Loader` ABC ``load_module`` methods (:meth:`!importlib.abc.Loader.load_module`, :meth:`!importlib.abc.InspectLoader.load_module`, :meth:`!importlib.abc.FileLoader.load_module`, @@ -2371,11 +2371,11 @@ Changes in the Python API 3.3.3. * The :attr:`~cgi.FieldStorage.file` attribute is now automatically closed when - the creating :class:`cgi.FieldStorage` instance is garbage collected. If you - were pulling the file object out separately from the :class:`cgi.FieldStorage` + the creating :class:`!cgi.FieldStorage` instance is garbage collected. If you + were pulling the file object out separately from the :class:`!cgi.FieldStorage` instance and not keeping the instance alive, then you should either store the - entire :class:`cgi.FieldStorage` instance or read the contents of the file - before the :class:`cgi.FieldStorage` instance is garbage collected. + entire :class:`!cgi.FieldStorage` instance or read the contents of the file + before the :class:`!cgi.FieldStorage` instance is garbage collected. * Calling ``read`` or ``write`` on a closed SSL socket now raises an informative :exc:`ValueError` rather than the previous more mysterious @@ -2434,7 +2434,7 @@ Changes in the Python API matched the input type, so if your code was depending on the return value being, say, a ``bytearray``, you will need to change your code. -* :mod:`audioop` functions now raise an error immediately if passed string +* :mod:`!audioop` functions now raise an error immediately if passed string input, instead of failing randomly later on (:issue:`16685`). * The new *convert_charrefs* argument to :class:`~html.parser.HTMLParser` diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 14b6425cea699e..ae6affcab664c6 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -478,7 +478,7 @@ not make an additional system call:: PEP 475: Retry system calls failing with EINTR ---------------------------------------------- -An :py:data:`errno.EINTR` error code is returned whenever a system call, that +An :py:const:`errno.EINTR` error code is returned whenever a system call, that is waiting for I/O, is interrupted by a signal. Previously, Python would raise :exc:`InterruptedError` in such cases. This meant that, when writing a Python application, the developer had two choices: @@ -527,7 +527,7 @@ by a signal: :func:`~os.writev`; * special cases: :func:`os.close` and :func:`os.dup2` now ignore - :py:data:`~errno.EINTR` errors; the syscall is not retried (see the PEP + :py:const:`~errno.EINTR` errors; the syscall is not retried (see the PEP for the rationale); * :mod:`select` functions: :func:`devpoll.poll() `, @@ -921,7 +921,7 @@ and improves their substitutability for lists. Docstrings produced by :func:`~collections.namedtuple` can now be updated:: Point = namedtuple('Point', ['x', 'y']) - Point.__doc__ += ': Cartesian coodinate' + Point.__doc__ += ': Cartesian coordinate' Point.x.__doc__ = 'abscissa' Point.y.__doc__ = 'ordinate' @@ -1045,8 +1045,8 @@ not just sequences. (Contributed by Serhiy Storchaka in :issue:`23171`.) curses ------ -The new :func:`~curses.update_lines_cols` function updates the :envvar:`LINES` -and :envvar:`COLS` environment variables. This is useful for detecting +The new :func:`~curses.update_lines_cols` function updates the :data:`LINES` +and :data:`COLS` module variables. This is useful for detecting manual screen resizing. (Contributed by Arnon Yaari in :issue:`4254`.) @@ -1252,7 +1252,7 @@ Oberkirch in :issue:`21800`.) imghdr ------ -The :func:`~imghdr.what` function now recognizes the +The :func:`~!imghdr.what` function now recognizes the `OpenEXR `_ format (contributed by Martin Vignali and Claudiu Popa in :issue:`20295`), and the `WebP `_ format @@ -1498,7 +1498,7 @@ use ``/dev/urandom`` and avoiding failures due to potential file descriptor exhaustion. (Contributed by Victor Stinner in :issue:`22181`.) New :func:`~os.get_blocking` and :func:`~os.set_blocking` functions allow -getting and setting a file descriptor's blocking mode (:data:`~os.O_NONBLOCK`.) +getting and setting a file descriptor's blocking mode (:const:`~os.O_NONBLOCK`.) (Contributed by Victor Stinner in :issue:`22054`.) The :func:`~os.truncate` and :func:`~os.ftruncate` functions are now supported @@ -1783,7 +1783,7 @@ the TLS handshake. The new :meth:`SSLSocket.selected_alpn_protocol() ` returns the protocol that was selected during the TLS handshake. -The :data:`~ssl.HAS_ALPN` flag indicates whether ALPN support is present. +The :const:`~ssl.HAS_ALPN` flag indicates whether ALPN support is present. Other Changes @@ -2095,7 +2095,7 @@ accepts ``"x"`` to request exclusive creation. Other module-level changes ========================== -Many functions in the :mod:`mmap`, :mod:`ossaudiodev`, :mod:`socket`, +Many functions in the :mod:`mmap`, :mod:`!ossaudiodev`, :mod:`socket`, :mod:`ssl`, and :mod:`codecs` modules now accept writable :term:`bytes-like objects `. (Contributed by Serhiy Storchaka in :issue:`23001`.) @@ -2192,7 +2192,7 @@ encode error with ``\N{...}`` escapes. (Contributed by Serhiy Storchaka in :issue:`19676`.) A new :c:func:`PyErr_FormatV` function similar to :c:func:`PyErr_Format`, -but accepts a ``va_list`` argument. +but accepts a :c:type:`va_list` argument. (Contributed by Antoine Pitrou in :issue:`18711`.) A new :c:data:`PyExc_RecursionError` exception. @@ -2476,7 +2476,7 @@ Changes in the Python API in Python 3.5, all old ``.pyo`` files from previous versions of Python are invalid regardless of this PEP. -* The :mod:`socket` module now exports the :data:`~socket.CAN_RAW_FD_FRAMES` +* The :mod:`socket` module now exports the :const:`~socket.CAN_RAW_FD_FRAMES` constant on linux 3.6 and greater. * The :func:`ssl.cert_time_to_seconds` function now interprets the input time @@ -2512,7 +2512,7 @@ Changes in the Python API Changes in the C API -------------------- -* The undocumented :c:member:`~PyMemoryViewObject.format` member of the +* The undocumented :c:member:`!format` member of the (non-public) :c:type:`PyMemoryViewObject` structure has been removed. All extensions relying on the relevant parts in ``memoryobject.h`` must be rebuilt. @@ -2520,7 +2520,7 @@ Changes in the C API * The :c:type:`PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. -* Removed non-documented macro :c:macro:`PyObject_REPR` which leaked references. +* Removed non-documented macro :c:macro:`!PyObject_REPR()` which leaked references. Use format character ``%R`` in :c:func:`PyUnicode_FromFormat`-like functions to format the :func:`repr` of the object. (Contributed by Serhiy Storchaka in :issue:`22453`.) @@ -2533,7 +2533,7 @@ Changes in the C API * As part of the :pep:`492` implementation, the ``tp_reserved`` slot of :c:type:`PyTypeObject` was replaced with a - :c:member:`tp_as_async` slot. Refer to :ref:`coro-objects` for + :c:member:`~PyTypeObject.tp_as_async` slot. Refer to :ref:`coro-objects` for new types, structures and functions. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3a681754e25dd7..5a3cea0ec87cb2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -650,8 +650,8 @@ compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: * Detect writes before the start of a buffer (buffer underflows) * Detect writes after the end of a buffer (buffer overflows) * Check that the :term:`GIL ` is held when allocator - functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and - :c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. + functions of :c:macro:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and + :c:macro:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. Checking if the GIL is held is also a new feature of Python 3.6. @@ -1388,7 +1388,7 @@ are treated as punctuation. site ---- -When specifying paths to add to :attr:`sys.path` in a ``.pth`` file, +When specifying paths to add to :data:`sys.path` in a ``.pth`` file, you may now specify file paths on top of directories (e.g. zip files). (Contributed by Wolfgang Langner in :issue:`26587`). @@ -1404,7 +1404,7 @@ socket ------ The :func:`~socket.socket.ioctl` function now supports the -:data:`~socket.SIO_LOOPBACK_FAST_PATH` control code. +:const:`~socket.SIO_LOOPBACK_FAST_PATH` control code. (Contributed by Daniel Stokes in :issue:`26536`.) The :meth:`~socket.socket.getsockopt` constants ``SO_DOMAIN``, @@ -1416,7 +1416,7 @@ The :meth:`~socket.socket.setsockopt` now supports the (Contributed by Christian Heimes in :issue:`27744`.) The socket module now supports the address family -:data:`~socket.AF_ALG` to interface with Linux Kernel crypto API. ``ALG_*``, +:const:`~socket.AF_ALG` to interface with Linux Kernel crypto API. ``ALG_*``, ``SOL_ALG`` and :meth:`~socket.socket.sendmsg_afalg` were added. (Contributed by Christian Heimes in :issue:`27744` with support from Victor Stinner.) @@ -1520,7 +1520,7 @@ rather than the version that is being emulated for the process telnetlib --------- -:class:`~telnetlib.Telnet` is now a context manager (contributed by +:class:`!telnetlib.Telnet` is now a context manager (contributed by Stéphane Wirtel in :issue:`25485`). @@ -1822,7 +1822,7 @@ Optimizations up to 80% faster. (Contributed by Josh Snider in :issue:`26574`). * Allocator functions of the :c:func:`PyMem_Malloc` domain - (:c:data:`PYMEM_DOMAIN_MEM`) now use the :ref:`pymalloc memory allocator + (:c:macro:`PYMEM_DOMAIN_MEM`) now use the :ref:`pymalloc memory allocator ` instead of :c:func:`malloc` function of the C library. The pymalloc allocator is optimized for objects smaller or equal to 512 bytes with a short lifetime, and use :c:func:`malloc` for larger memory blocks. @@ -1874,8 +1874,8 @@ Build and C API Changes (Original patch by Alecsandru Patrascu of Intel in :issue:`26359`.) * The :term:`GIL ` must now be held when allocator - functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and - :c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. + functions of :c:macro:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and + :c:macro:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. * New :c:func:`Py_FinalizeEx` API which indicates if flushing buffered data failed. @@ -2010,7 +2010,7 @@ been deprecated in previous versions of Python in favour of :meth:`importlib.abc.Loader.exec_module`. The :class:`importlib.machinery.WindowsRegistryFinder` class is now -deprecated. As of 3.6.0, it is still added to :attr:`sys.meta_path` by +deprecated. As of 3.6.0, it is still added to :data:`sys.meta_path` by default (on Windows), but this may change in future releases. os @@ -2049,7 +2049,7 @@ of OpenSSL. Other features are deprecated in favor of a different API. tkinter ~~~~~~~ -The :mod:`tkinter.tix` module is now deprecated. :mod:`tkinter` users +The :mod:`!tkinter.tix` module is now deprecated. :mod:`tkinter` users should use :mod:`tkinter.ttk` instead. .. _whatsnew36-venv: @@ -2066,9 +2066,9 @@ environment. (Contributed by Brett Cannon in :issue:`25154`.) Deprecated functions and types of the C API ------------------------------------------- -Undocumented functions :c:func:`PyUnicode_AsEncodedObject`, -:c:func:`PyUnicode_AsDecodedObject`, :c:func:`PyUnicode_AsEncodedUnicode` -and :c:func:`PyUnicode_AsDecodedUnicode` are deprecated now. +Undocumented functions :c:func:`!PyUnicode_AsEncodedObject`, +:c:func:`!PyUnicode_AsDecodedObject`, :c:func:`!PyUnicode_AsEncodedUnicode` +and :c:func:`!PyUnicode_AsDecodedUnicode` are deprecated now. Use the :ref:`generic codec based API ` instead. @@ -2160,14 +2160,15 @@ Changes in the Python API * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** argument is not set. Previously only ``NULL`` was returned. -* The format of the ``co_lnotab`` attribute of code objects changed to support +* The format of the :attr:`~codeobject.co_lnotab` attribute of code objects + changed to support a negative line number delta. By default, Python does not emit bytecode with - a negative line number delta. Functions using ``frame.f_lineno``, + a negative line number delta. Functions using :attr:`frame.f_lineno`, ``PyFrame_GetLineNumber()`` or ``PyCode_Addr2Line()`` are not affected. - Functions directly decoding ``co_lnotab`` should be updated to use a signed + Functions directly decoding :attr:`!co_lnotab` should be updated to use a signed 8-bit integer type for the line number delta, but this is only required to support applications using a negative line number delta. See - ``Objects/lnotab_notes.txt`` for the ``co_lnotab`` format and how to decode + ``Objects/lnotab_notes.txt`` for the :attr:`!co_lnotab` format and how to decode it, and see the :pep:`511` for the rationale. * The functions in the :mod:`compileall` module now return booleans instead @@ -2185,7 +2186,7 @@ Changes in the Python API * The following modules have had missing APIs added to their :attr:`__all__` attributes to match the documented APIs: - :mod:`calendar`, :mod:`cgi`, :mod:`csv`, + :mod:`calendar`, :mod:`!cgi`, :mod:`csv`, :mod:`~xml.etree.ElementTree`, :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`mailbox`, :mod:`mimetypes`, :mod:`optparse`, :mod:`plistlib`, :mod:`smtpd`, @@ -2211,7 +2212,7 @@ Changes in the Python API the exception will stop a single-threaded server. (Contributed by Martin Panter in :issue:`23430`.) -* :func:`spwd.getspnam` now raises a :exc:`PermissionError` instead of +* :func:`!spwd.getspnam` now raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user doesn't have privileges. * The :meth:`socket.socket.close` method now raises an exception if @@ -2274,7 +2275,7 @@ Changes in the Python API :class:`~collections.OrderedDict`. (Contributed by Steve Holden in :issue:`27842`.) -* The :const:`crypt.METHOD_CRYPT` will no longer be added to ``crypt.methods`` +* The :const:`!crypt.METHOD_CRYPT` will no longer be added to ``crypt.methods`` if unsupported by the platform. (Contributed by Victor Stinner in :issue:`25287`.) @@ -2455,7 +2456,7 @@ query parameter separators in :func:`urllib.parse.parse_qs` and :func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with newer W3C recommendations, this has been changed to allow only a single separator key, with ``&`` as the default. This change also affects -:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +:func:`!cgi.parse` and :func:`!cgi.parse_multipart` as they use the affected functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 28f22836d8d09e..7a74f9c1685c31 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -848,10 +848,10 @@ alternative to script path. (Contributed by Sanyam Khurana in :issue:`21862`.) crypt ----- -The :mod:`crypt` module now supports the Blowfish hashing method. +The :mod:`!crypt` module now supports the Blowfish hashing method. (Contributed by Serhiy Storchaka in :issue:`31664`.) -The :func:`~crypt.mksalt` function now allows specifying the number of rounds +The :func:`~!crypt.mksalt` function now allows specifying the number of rounds for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) @@ -1133,7 +1133,7 @@ The MIME type of .bmp has been changed from ``'image/x-ms-bmp'`` to msilib ------ -The new :meth:`Database.Close() ` method can be used +The new :meth:`!Database.Close()` method can be used to close the :abbr:`MSI` database. (Contributed by Berker Peksag in :issue:`20486`.) @@ -1280,13 +1280,13 @@ This function should be used instead of :func:`os.close` for better compatibility across platforms. (Contributed by Christian Heimes in :issue:`32454`.) -The :mod:`socket` module now exposes the :data:`socket.TCP_CONGESTION` -(Linux 2.6.13), :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37), and -:data:`socket.TCP_NOTSENT_LOWAT` (Linux 3.12) constants. +The :mod:`socket` module now exposes the :const:`socket.TCP_CONGESTION` +(Linux 2.6.13), :const:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37), and +:const:`socket.TCP_NOTSENT_LOWAT` (Linux 3.12) constants. (Contributed by Omar Sandoval in :issue:`26273` and Nathaniel J. Smith in :issue:`29728`.) -Support for :data:`socket.AF_VSOCK` sockets has been added to allow +Support for :const:`socket.AF_VSOCK` sockets has been added to allow communication between virtual machines and their hosts. (Contributed by Cathy Avery in :issue:`27584`.) @@ -1394,7 +1394,7 @@ subprocess The :func:`subprocess.run` function accepts the new *capture_output* keyword argument. When true, stdout and stderr will be captured. -This is equivalent to passing :data:`subprocess.PIPE` as *stdout* and +This is equivalent to passing :const:`subprocess.PIPE` as *stdout* and *stderr* arguments. (Contributed by Bo Bayles in :issue:`32102`.) @@ -1453,12 +1453,12 @@ time New clock identifiers have been added: -* :data:`time.CLOCK_BOOTTIME` (Linux): Identical to - :data:`time.CLOCK_MONOTONIC`, except it also includes any time that the +* :const:`time.CLOCK_BOOTTIME` (Linux): Identical to + :const:`time.CLOCK_MONOTONIC`, except it also includes any time that the system is suspended. -* :data:`time.CLOCK_PROF` (FreeBSD, NetBSD and OpenBSD): High-resolution +* :const:`time.CLOCK_PROF` (FreeBSD, NetBSD and OpenBSD): High-resolution per-process CPU timer. -* :data:`time.CLOCK_UPTIME` (FreeBSD, OpenBSD): Time whose absolute value is +* :const:`time.CLOCK_UPTIME` (FreeBSD, OpenBSD): Time whose absolute value is the time the system has been running and not suspended, providing accurate uptime measurement. @@ -1547,7 +1547,7 @@ adding ``~`` to the set of characters that are never quoted by default. uu -- -The :func:`uu.encode` function now accepts an optional *backtick* +The :func:`!uu.encode` function now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) @@ -1580,13 +1580,13 @@ The initialization of the default warnings filters has changed as follows: * warnings filters enabled via the command line or the environment now have the following order of precedence: - * the ``BytesWarning`` filter for :option:`-b` (or ``-bb``) - * any filters specified with the :option:`-W` option - * any filters specified with the :envvar:`PYTHONWARNINGS` environment - variable - * any other CPython specific filters (e.g. the ``default`` filter added - for the new ``-X dev`` mode) - * any implicit filters defined directly by the warnings machinery + * the ``BytesWarning`` filter for :option:`-b` (or ``-bb``) + * any filters specified with the :option:`-W` option + * any filters specified with the :envvar:`PYTHONWARNINGS` environment + variable + * any other CPython specific filters (e.g. the ``default`` filter added + for the new ``-X dev`` mode) + * any implicit filters defined directly by the warnings machinery * in :ref:`CPython debug builds `, all warnings are now displayed by default (the implicit filter list is empty) @@ -1674,10 +1674,10 @@ The new :c:func:`import__find__load__start` and module imports. (Contributed by Christian Heimes in :issue:`31574`.) -The fields :c:member:`name` and :c:member:`doc` of structures +The fields :c:member:`!name` and :c:member:`!doc` of structures :c:type:`PyMemberDef`, :c:type:`PyGetSetDef`, :c:type:`PyStructSequence_Field`, :c:type:`PyStructSequence_Desc`, -and :c:type:`wrapperbase` are now of type ``const char *`` rather of +and :c:struct:`wrapperbase` are now of type ``const char *`` rather of ``char *``. (Contributed by Serhiy Storchaka in :issue:`28761`.) The result of :c:func:`PyUnicode_AsUTF8AndSize` and :c:func:`PyUnicode_AsUTF8` @@ -1891,7 +1891,7 @@ Other CPython Implementation Changes * Trace hooks may now opt out of receiving the ``line`` and opt into receiving the ``opcode`` events from the interpreter by setting the corresponding new - ``f_trace_lines`` and ``f_trace_opcodes`` attributes on the + :attr:`~frame.f_trace_lines` and :attr:`~frame.f_trace_opcodes` attributes on the frame being traced. (Contributed by Nick Coghlan in :issue:`31344`.) * Fixed some consistency problems with namespace package module attributes. @@ -1940,8 +1940,8 @@ Deprecated Python modules, functions and methods aifc ---- -:func:`aifc.openfp` has been deprecated and will be removed in Python 3.9. -Use :func:`aifc.open` instead. +:func:`!aifc.openfp` has been deprecated and will be removed in Python 3.9. +Use :func:`!aifc.open` instead. (Contributed by Brian Curtin in :issue:`31985`.) @@ -2061,8 +2061,8 @@ ssl sunau ----- -:func:`sunau.openfp` has been deprecated and will be removed in Python 3.9. -Use :func:`sunau.open` instead. +:func:`!sunau.openfp` has been deprecated and will be removed in Python 3.9. +Use :func:`!sunau.open` instead. (Contributed by Brian Curtin in :issue:`31985`.) @@ -2144,9 +2144,9 @@ The following features and APIs have been removed from Python 3.7: * Removed support of the *exclude* argument in :meth:`tarfile.TarFile.add`. It was deprecated in Python 2.7 and 3.2. Use the *filter* argument instead. -* The ``splitunc()`` function in the :mod:`ntpath` module was deprecated in - Python 3.1, and has now been removed. Use the :func:`~os.path.splitdrive` - function instead. +* The :func:`!ntpath.splitunc` function was deprecated in + Python 3.1, and has now been removed. Use :func:`~os.path.splitdrive` + instead. * :func:`collections.namedtuple` no longer supports the *verbose* parameter or ``_source`` attribute which showed the generated source code for the @@ -2495,12 +2495,12 @@ either in embedding applications, or in CPython itself. :issue:`22257`, and further updated by Nick, Eric, and Victor Stinner in a number of other issues). Some known details affected: -* :c:func:`PySys_AddWarnOptionUnicode` is not currently usable by embedding +* :c:func:`!PySys_AddWarnOptionUnicode` is not currently usable by embedding applications due to the requirement to create a Unicode object prior to - calling ``Py_Initialize``. Use :c:func:`PySys_AddWarnOption` instead. + calling ``Py_Initialize``. Use :c:func:`!PySys_AddWarnOption` instead. * warnings filters added by an embedding application with - :c:func:`PySys_AddWarnOption` should now more consistently take precedence + :c:func:`!PySys_AddWarnOption` should now more consistently take precedence over the default filters set by the interpreter Due to changes in the way the default warnings filters are configured, @@ -2567,7 +2567,7 @@ query parameter separators in :func:`urllib.parse.parse_qs` and :func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with newer W3C recommendations, this has been changed to allow only a single separator key, with ``&`` as the default. This change also affects -:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +:func:`!cgi.parse` and :func:`!cgi.parse_multipart` as they use the affected functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 85e088b64acb2d..4574702b1a600f 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -123,7 +123,7 @@ There is a new function parameter syntax ``/`` to indicate that some function parameters must be specified positionally and cannot be used as keyword arguments. This is the same notation shown by ``help()`` for C functions annotated with Larry Hastings' -:ref:`Argument Clinic ` tool. +`Argument Clinic `__ tool. In the following example, parameters *a* and *b* are positional-only, while *c* or *d* can be positional or keyword, and *e* or *f* are @@ -404,7 +404,7 @@ Other Language Changes or :meth:`~object.__complex__` is not available. (Contributed by Serhiy Storchaka in :issue:`20092`.) -* Added support of ``\N{name}`` escapes in :mod:`regular expressions `:: +* Added support of :samp:`\\N\\{{name}\\}` escapes in :mod:`regular expressions `:: >>> notice = 'Copyright © 2019' >>> copyright_year_pattern = re.compile(r'\N{copyright sign}\s*(\d{4})') @@ -947,7 +947,7 @@ This made it difficult to update, experiment with, or teach the various logging configuration options using the interactive prompt or a Jupyter notebook. -(Suggested by Raymond Hettinger, implemented by Dong-hee Na, and +(Suggested by Raymond Hettinger, implemented by Donghee Na, and reviewed by Vinay Sajip in :issue:`33897`.) @@ -1305,7 +1305,7 @@ Zackery Spytz in :issue:`25451`.) time ---- -Added new clock :data:`~time.CLOCK_UPTIME_RAW` for macOS 10.12. +Added new clock :const:`~time.CLOCK_UPTIME_RAW` for macOS 10.12. (Contributed by Joannah Nanjekye in :issue:`35702`.) @@ -1574,12 +1574,12 @@ Build and C API Changes * :c:func:`Py_INCREF`, :c:func:`Py_DECREF` * :c:func:`Py_XINCREF`, :c:func:`Py_XDECREF` * :c:func:`PyObject_INIT`, :c:func:`PyObject_INIT_VAR` - * Private functions: :c:func:`_PyObject_GC_TRACK`, - :c:func:`_PyObject_GC_UNTRACK`, :c:func:`_Py_Dealloc` + * Private functions: :c:func:`!_PyObject_GC_TRACK`, + :c:func:`!_PyObject_GC_UNTRACK`, :c:func:`!_Py_Dealloc` (Contributed by Victor Stinner in :issue:`35059`.) -* The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +* The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. (Contributed by Victor Stinner in :issue:`35713`.) @@ -1628,7 +1628,7 @@ Build and C API Changes parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) -* :c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +* :c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). (Contributed by Victor Stinner in :issue:`38234`.) @@ -1653,7 +1653,7 @@ Deprecated deprecated and will be prohibited in Python 3.9. (Contributed by Elvis Pranskevichus in :issue:`34075`.) -* The :meth:`__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, +* The :meth:`~object.__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` and :class:`fileinput.FileInput` have been deprecated. @@ -1714,7 +1714,7 @@ Deprecated * The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` has been deprecated. - (Contributed by Dong-hee Na in :issue:`35283`.) + (Contributed by Donghee Na in :issue:`35283`.) * Many builtin and extension functions that take integer arguments will now emit a deprecation warning for :class:`~decimal.Decimal`\ s, @@ -1774,7 +1774,7 @@ The following features and APIs have been removed from Python 3.8: to help eliminate confusion as to what Python interpreter the ``pyvenv`` script is tied to. (Contributed by Brett Cannon in :issue:`25427`.) -* ``parse_qs``, ``parse_qsl``, and ``escape`` are removed from the :mod:`cgi` +* ``parse_qs``, ``parse_qsl``, and ``escape`` are removed from the :mod:`!cgi` module. They are deprecated in Python 3.2 or older. They should be imported from the ``urllib.parse`` and ``html`` modules instead. @@ -1839,18 +1839,18 @@ Changes in Python behavior classes will affect their string representation. (Contributed by Serhiy Storchaka in :issue:`36793`.) -* On AIX, :attr:`sys.platform` doesn't contain the major version anymore. +* On AIX, :data:`sys.platform` doesn't contain the major version anymore. It is always ``'aix'``, instead of ``'aix3'`` .. ``'aix7'``. Since older Python versions include the version number, so it is recommended to always use ``sys.platform.startswith('aix')``. (Contributed by M. Felt in :issue:`36588`.) -* :c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +* :c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. If this - behavior is not desired, guard the call by checking :c:func:`_Py_IsFinalizing` - or :c:func:`sys.is_finalizing`. + behavior is not desired, guard the call by checking :c:func:`!_Py_IsFinalizing` + or :func:`sys.is_finalizing`. (Contributed by Joannah Nanjekye in :issue:`36475`.) @@ -2021,7 +2021,7 @@ Changes in the C API *cf_flags*. (Contributed by Guido van Rossum in :issue:`35766`.) -* The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +* The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. (Contributed by Victor Stinner in :issue:`36728`.) @@ -2061,8 +2061,8 @@ Changes in the C API * Remove :c:macro:`Py_INCREF` on the type object after allocating an instance - if any. - This may happen after calling :c:func:`PyObject_New`, - :c:func:`PyObject_NewVar`, :c:func:`PyObject_GC_New`, + This may happen after calling :c:macro:`PyObject_New`, + :c:macro:`PyObject_NewVar`, :c:func:`PyObject_GC_New`, :c:func:`PyObject_GC_NewVar`, or any other custom allocator that uses :c:func:`PyObject_Init` or :c:func:`PyObject_INIT`. @@ -2116,12 +2116,12 @@ Changes in the C API extension types across feature releases, anymore. A :c:type:`PyTypeObject` exported by a third-party extension module is supposed to have all the slots expected in the current Python version, including - :c:member:`~PyTypeObject.tp_finalize` (:const:`Py_TPFLAGS_HAVE_FINALIZE` + :c:member:`~PyTypeObject.tp_finalize` (:c:macro:`Py_TPFLAGS_HAVE_FINALIZE` is not checked anymore before reading :c:member:`~PyTypeObject.tp_finalize`). (Contributed by Antoine Pitrou in :issue:`32388`.) -* The functions :c:func:`PyNode_AddChild` and :c:func:`PyParser_AddToken` now accept +* The functions :c:func:`!PyNode_AddChild` and :c:func:`!PyParser_AddToken` now accept two additional ``int`` arguments *end_lineno* and *end_col_offset*. * The :file:`libpython38.a` file to allow MinGW tools to link directly against @@ -2251,7 +2251,7 @@ query parameter separators in :func:`urllib.parse.parse_qs` and :func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with newer W3C recommendations, this has been changed to allow only a single separator key, with ``&`` as the default. This change also affects -:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +:func:`!cgi.parse` and :func:`!cgi.parse_multipart` as they use the affected functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index fd86db96302356..cb2482ee48d7fa 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -44,7 +44,6 @@ This article explains the new features in Python 3.9, compared to 3.8. Python 3.9 was released on October 5, 2020. - For full details, see the :ref:`changelog `. .. seealso:: @@ -82,12 +81,12 @@ Interpreter improvements: * a number of Python builtins (range, tuple, set, frozenset, list, dict) are now sped up using :pep:`590` vectorcall; * garbage collection does not block on resurrected objects; -* a number of Python modules (:mod:`_abc`, :mod:`audioop`, :mod:`_bz2`, - :mod:`_codecs`, :mod:`_contextvars`, :mod:`_crypt`, :mod:`_functools`, +* a number of Python modules (:mod:`_abc`, :mod:`!audioop`, :mod:`_bz2`, + :mod:`_codecs`, :mod:`_contextvars`, :mod:`!_crypt`, :mod:`_functools`, :mod:`_json`, :mod:`_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, :mod:`time`, :mod:`_weakref`) now use multiphase initialization as defined by PEP 489; -* a number of standard library modules (:mod:`audioop`, :mod:`ast`, :mod:`grp`, +* a number of standard library modules (:mod:`!audioop`, :mod:`ast`, :mod:`grp`, :mod:`_hashlib`, :mod:`pwd`, :mod:`_posixsubprocess`, :mod:`random`, :mod:`select`, :mod:`struct`, :mod:`termios`, :mod:`zlib`) are now using the stable ABI defined by PEP 384. @@ -415,7 +414,7 @@ datetime The :meth:`~datetime.date.isocalendar()` of :class:`datetime.date` and :meth:`~datetime.datetime.isocalendar()` of :class:`datetime.datetime` methods now returns a :func:`~collections.namedtuple` instead of a :class:`tuple`. -(Contributed by Dong-hee Na in :issue:`24416`.) +(Contributed by Donghee Na in :issue:`24416`.) distutils --------- @@ -427,16 +426,16 @@ digests. It skips MD5 on platforms that block MD5 digest. fcntl ----- -Added constants :data:`~fcntl.F_OFD_GETLK`, :data:`~fcntl.F_OFD_SETLK` -and :data:`~fcntl.F_OFD_SETLKW`. -(Contributed by Dong-hee Na in :issue:`38602`.) +Added constants :const:`~fcntl.F_OFD_GETLK`, :const:`~fcntl.F_OFD_SETLK` +and :const:`~fcntl.F_OFD_SETLKW`. +(Contributed by Donghee Na in :issue:`38602`.) ftplib ------- :class:`~ftplib.FTP` and :class:`~ftplib.FTP_TLS` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of -a non-blocking socket. (Contributed by Dong-hee Na in :issue:`39259`.) +a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) gc -- @@ -468,7 +467,7 @@ http ---- HTTP status codes ``103 EARLY_HINTS``, ``418 IM_A_TEAPOT`` and ``425 TOO_EARLY`` are added to -:class:`http.HTTPStatus`. (Contributed by Dong-hee Na in :issue:`39509` and Ross Rhodes in :issue:`39507`.) +:class:`http.HTTPStatus`. (Contributed by Donghee Na in :issue:`39509` and Ross Rhodes in :issue:`39507`.) IDLE and idlelib ---------------- @@ -509,14 +508,14 @@ an optional *timeout* parameter for their constructors. Also, the :meth:`~imaplib.IMAP4.open` method now has an optional *timeout* parameter with this change. The overridden methods of :class:`~imaplib.IMAP4_SSL` and :class:`~imaplib.IMAP4_stream` were applied to this change. -(Contributed by Dong-hee Na in :issue:`38615`.) +(Contributed by Donghee Na in :issue:`38615`.) :meth:`imaplib.IMAP4.unselect` is added. :meth:`imaplib.IMAP4.unselect` frees server's resources associated with the selected mailbox and returns the server to the authenticated state. This command performs the same actions as :meth:`imaplib.IMAP4.close`, except that no messages are permanently removed from the currently -selected mailbox. (Contributed by Dong-hee Na in :issue:`40375`.) +selected mailbox. (Contributed by Donghee Na in :issue:`40375`.) importlib --------- @@ -586,18 +585,18 @@ queue. nntplib ------- -:class:`~nntplib.NNTP` and :class:`~nntplib.NNTP_SSL` now raise a :class:`ValueError` +:class:`~!nntplib.NNTP` and :class:`~!nntplib.NNTP_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of -a non-blocking socket. (Contributed by Dong-hee Na in :issue:`39259`.) +a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) os -- -Added :data:`~os.CLD_KILLED` and :data:`~os.CLD_STOPPED` for :attr:`si_code`. -(Contributed by Dong-hee Na in :issue:`38493`.) +Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`si_code`. +(Contributed by Donghee Na in :issue:`38493`.) Exposed the Linux-specific :func:`os.pidfd_open` (:issue:`38692`) and -:data:`os.P_PIDFD` (:issue:`38713`) for process management with file +:const:`os.P_PIDFD` (:issue:`38713`) for process management with file descriptors. The :func:`os.unsetenv` function is now also available on Windows. @@ -629,7 +628,7 @@ poplib :class:`~poplib.POP3` and :class:`~poplib.POP3_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of -a non-blocking socket. (Contributed by Dong-hee Na in :issue:`39259`.) +a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) pprint ------ @@ -661,19 +660,19 @@ smtplib :class:`~smtplib.SMTP` and :class:`~smtplib.SMTP_SSL` now raise a :class:`ValueError` if the given timeout for their constructor is zero to prevent the creation of -a non-blocking socket. (Contributed by Dong-hee Na in :issue:`39259`.) +a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) :class:`~smtplib.LMTP` constructor now has an optional *timeout* parameter. -(Contributed by Dong-hee Na in :issue:`39329`.) +(Contributed by Donghee Na in :issue:`39329`.) socket ------ -The :mod:`socket` module now exports the :data:`~socket.CAN_RAW_JOIN_FILTERS` +The :mod:`socket` module now exports the :const:`~socket.CAN_RAW_JOIN_FILTERS` constant on Linux 4.1 and greater. (Contributed by Stefan Tatschner and Zackery Spytz in :issue:`25780`.) -The socket module now supports the :data:`~socket.CAN_J1939` protocol on +The socket module now supports the :const:`~socket.CAN_J1939` protocol on platforms that support it. (Contributed by Karl Ding in :issue:`40291`.) The socket module now has the :func:`socket.send_fds` and @@ -692,13 +691,13 @@ which has nanosecond resolution, rather than sys --- -Added a new :attr:`sys.platlibdir` attribute: name of the platform-specific +Added a new :data:`sys.platlibdir` attribute: name of the platform-specific library directory. It is used to build the path of standard library and the paths of installed extension modules. It is equal to ``"lib"`` on most platforms. On Fedora and SuSE, it is equal to ``"lib64"`` on 64-bit platforms. (Contributed by Jan MatÄ›jek, MatÄ›j Cepl, Charalampos Stratakis and Victor Stinner in :issue:`1294959`.) -Previously, :attr:`sys.stderr` was block-buffered when non-interactive. Now +Previously, :data:`sys.stderr` was block-buffered when non-interactive. Now ``stderr`` defaults to always being line-buffered. (Contributed by Jendrik Seipp in :issue:`13601`.) @@ -777,7 +776,7 @@ Optimizations * A number of Python builtins (:class:`range`, :class:`tuple`, :class:`set`, :class:`frozenset`, :class:`list`, :class:`dict`) are now sped up by using :pep:`590` vectorcall protocol. - (Contributed by Dong-hee Na, Mark Shannon, Jeroen Demeyer and Petr Viktorin in :issue:`37207`.) + (Contributed by Donghee Na, Mark Shannon, Jeroen Demeyer and Petr Viktorin in :issue:`37207`.) * Optimized :func:`~set.difference_update` for the case when the other set is much larger than the base set. @@ -791,7 +790,7 @@ Optimizations * :term:`floor division` of float operation now has a better performance. Also the message of :exc:`ZeroDivisionError` for this operation is updated. - (Contributed by Dong-hee Na in :issue:`39434`.) + (Contributed by Donghee Na in :issue:`39434`.) * Decoding short ASCII strings with UTF-8 and ascii codecs is now about 15% faster. (Contributed by Inada Naoki in :issue:`37348`.) @@ -870,9 +869,9 @@ Deprecated users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. -* The Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, - :c:func:`PyParser_SimpleParseStringFlagsFilename`, - :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` +* The Public C API functions :c:func:`!PyParser_SimpleParseStringFlags`, + :c:func:`!PyParser_SimpleParseStringFlagsFilename`, + :c:func:`!PyParser_SimpleParseFileFlags` and :c:func:`!PyNode_Compile` are deprecated and will be removed in Python 3.10 together with the old parser. * Using :data:`NotImplemented` in a boolean context has been deprecated, @@ -923,10 +922,10 @@ Deprecated (Contributed by Batuhan Taskaya in :issue:`39639` and :issue:`39969` and Serhiy Storchaka in :issue:`39988`.) -* The :c:func:`PyEval_InitThreads` and :c:func:`PyEval_ThreadsInitialized` +* The :c:func:`!PyEval_InitThreads` and :c:func:`!PyEval_ThreadsInitialized` functions are now deprecated and will be removed in Python 3.11. Calling - :c:func:`PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized - by :c:func:`Py_Initialize()` since Python 3.7. + :c:func:`!PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized + by :c:func:`Py_Initialize` since Python 3.7. (Contributed by Victor Stinner in :issue:`39877`.) * Passing ``None`` as the first argument to the :func:`shlex.split` function @@ -935,10 +934,10 @@ Deprecated * :func:`smtpd.MailmanProxy` is now deprecated as it is unusable without an external module, ``mailman``. (Contributed by Samuel Colvin in :issue:`35800`.) -* The :mod:`lib2to3` module now emits a :exc:`PendingDeprecationWarning`. +* The :mod:`!lib2to3` module now emits a :exc:`PendingDeprecationWarning`. Python 3.9 switched to a PEG parser (see :pep:`617`), and Python 3.10 may include new language syntax that is not parsable by lib2to3's LL(1) parser. - The ``lib2to3`` module may be removed from the standard library in a future + The :mod:`!lib2to3` module may be removed from the standard library in a future Python version. Consider third-party alternatives such as `LibCST`_ or `parso`_. (Contributed by Carl Meyer in :issue:`40360`.) @@ -956,12 +955,12 @@ Removed * The erroneous version at :data:`unittest.mock.__version__` has been removed. -* :class:`nntplib.NNTP`: ``xpath()`` and ``xgtitle()`` methods have been removed. +* :class:`!nntplib.NNTP`: ``xpath()`` and ``xgtitle()`` methods have been removed. These methods are deprecated since Python 3.3. Generally, these extensions are not supported or not enabled by NNTP server administrators. - For ``xgtitle()``, please use :meth:`nntplib.NNTP.descriptions` or - :meth:`nntplib.NNTP.description` instead. - (Contributed by Dong-hee Na in :issue:`39366`.) + For ``xgtitle()``, please use :meth:`!nntplib.NNTP.descriptions` or + :meth:`!nntplib.NNTP.description` instead. + (Contributed by Donghee Na in :issue:`39366`.) * :class:`array.array`: ``tostring()`` and ``fromstring()`` methods have been removed. They were aliases to ``tobytes()`` and ``frombytes()``, deprecated @@ -994,7 +993,7 @@ Removed * The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` has been removed. It was deprecated since Python 3.8. Use :meth:`~threading.Thread.is_alive()` instead. - (Contributed by Dong-hee Na in :issue:`37804`.) + (Contributed by Donghee Na in :issue:`37804`.) * Methods ``getchildren()`` and ``getiterator()`` of classes :class:`~xml.etree.ElementTree.ElementTree` and @@ -1084,7 +1083,7 @@ Changes in the Python API ``__VENV_PROMPT__`` is set to ``""``. * The :meth:`select.epoll.unregister` method no longer ignores the - :data:`~errno.EBADF` error. + :const:`~errno.EBADF` error. (Contributed by Victor Stinner in :issue:`39239`.) * The *compresslevel* parameter of :class:`bz2.BZ2File` became keyword-only, @@ -1115,9 +1114,9 @@ Changes in the Python API ``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``. (Contributed by Batuhan Taskaya in :issue:`39562`) -* ``array('u')`` now uses ``wchar_t`` as C type instead of ``Py_UNICODE``. +* ``array('u')`` now uses :c:type:`wchar_t` as C type instead of ``Py_UNICODE``. This change doesn't affect to its behavior because ``Py_UNICODE`` is alias - of ``wchar_t`` since Python 3.3. + of :c:type:`wchar_t` since Python 3.3. (Contributed by Inada Naoki in :issue:`34538`.) * The :func:`logging.getLogger` API now returns the root logger when passed @@ -1226,8 +1225,8 @@ Build Changes ============= * Added ``--with-platlibdir`` option to the ``configure`` script: name of the - platform-specific library directory, stored in the new :attr:`sys.platlibdir` - attribute. See :attr:`sys.platlibdir` attribute for more information. + platform-specific library directory, stored in the new :data:`sys.platlibdir` + attribute. See :data:`sys.platlibdir` attribute for more information. (Contributed by Jan MatÄ›jek, MatÄ›j Cepl, Charalampos Stratakis and Victor Stinner in :issue:`1294959`.) @@ -1276,7 +1275,7 @@ New Features * :pep:`573`: Added :c:func:`PyType_FromModuleAndSpec` to associate a module with a class; :c:func:`PyType_GetModule` and :c:func:`PyType_GetModuleState` to retrieve the module and its state; and - :c:data:`PyCMethod` and :c:data:`METH_METHOD` to allow a method to + :c:type:`PyCMethod` and :c:macro:`METH_METHOD` to allow a method to access the class it was defined in. (Contributed by Marcel Plch and Petr Viktorin in :issue:`38787`.) @@ -1315,7 +1314,7 @@ New Features * The :c:func:`PyModule_AddType` function is added to help adding a type to a module. - (Contributed by Dong-hee Na in :issue:`40024`.) + (Contributed by Donghee Na in :issue:`40024`.) * Added the functions :c:func:`PyObject_GC_IsTracked` and :c:func:`PyObject_GC_IsFinalized` to the public API to allow to query if @@ -1370,8 +1369,8 @@ Porting to Python 3.9 (Contributed by Victor Stinner in :issue:`40241`.) * The ``Py_UNICODE_COPY``, ``Py_UNICODE_FILL``, ``PyUnicode_WSTR_LENGTH``, - :c:func:`PyUnicode_FromUnicode`, :c:func:`PyUnicode_AsUnicode`, - ``_PyUnicode_AsUnicode``, and :c:func:`PyUnicode_AsUnicodeAndSize` are + :c:func:`!PyUnicode_FromUnicode`, :c:func:`!PyUnicode_AsUnicode`, + ``_PyUnicode_AsUnicode``, and :c:func:`!PyUnicode_AsUnicodeAndSize` are marked as deprecated in C. They have been deprecated by :pep:`393` since Python 3.3. (Contributed by Inada Naoki in :issue:`36346`.) @@ -1389,8 +1388,8 @@ Porting to Python 3.9 * :c:func:`PyObject_IS_GC` macro was converted to a function. * The :c:func:`PyObject_NEW` macro becomes an alias to the - :c:func:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro - becomes an alias to the :c:func:`PyObject_NewVar` macro. They no longer + :c:macro:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro + becomes an alias to the :c:macro:`PyObject_NewVar` macro. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. * :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: @@ -1559,7 +1558,7 @@ query parameter separators in :func:`urllib.parse.parse_qs` and :func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with newer W3C recommendations, this has been changed to allow only a single separator key, with ``&`` as the default. This change also affects -:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +:func:`!cgi.parse` and :func:`!cgi.parse_multipart` as they use the affected functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index bfee225791eee9..b9c19602653219 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.13.rst 3.12.rst 3.11.rst 3.10.rst diff --git a/Grammar/Tokens b/Grammar/Tokens index 096876fdd130f8..20bb803b7d58a6 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -56,17 +56,15 @@ COLONEQUAL ':=' EXCLAMATION '!' OP -AWAIT -ASYNC TYPE_IGNORE TYPE_COMMENT SOFT_KEYWORD FSTRING_START FSTRING_MIDDLE FSTRING_END +COMMENT +NL ERRORTOKEN # These aren't used by the C tokenizer but are needed for tokenize.py -COMMENT -NL ENCODING diff --git a/Grammar/python.gram b/Grammar/python.gram index 6361dcd0985b99..c9269b058bc7f6 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -19,8 +19,6 @@ _PyPegen_parse(Parser *p) result = eval_rule(p); } else if (p->start_rule == Py_func_type_input) { result = func_type_rule(p); - } else if (p->start_rule == Py_fstring_input) { - result = fstring_rule(p); } return result; @@ -89,7 +87,6 @@ file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) } interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(a, p->arena) } eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { _PyAST_Expression(a, p->arena) } func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { _PyAST_FunctionType(a, b, p->arena) } -fstring[expr_ty]: star_expressions # GENERAL STATEMENTS # ================== @@ -112,6 +109,7 @@ simple_stmts[asdl_stmt_seq*]: # will throw a SyntaxError. simple_stmt[stmt_ty] (memo): | assignment + | &"type" type_alias | e=star_expressions { _PyAST_Expr(e, EXTRA) } | &'return' return_stmt | &('import' | 'from') import_stmt @@ -126,11 +124,12 @@ simple_stmt[stmt_ty] (memo): | &'nonlocal' nonlocal_stmt compound_stmt[stmt_ty]: - | &('def' | '@' | ASYNC) function_def + | invalid_compound_stmt + | &('def' | '@' | 'async') function_def | &'if' if_stmt | &('class' | '@') class_def - | &('with' | ASYNC) with_stmt - | &('for' | ASYNC) for_stmt + | &('with' | 'async') with_stmt + | &('for' | 'async') for_stmt | &'try' try_stmt | &'while' while_stmt | match_stmt @@ -252,11 +251,11 @@ class_def[stmt_ty]: class_def_raw[stmt_ty]: | invalid_class_def_raw - | 'class' a=NAME b=['(' z=[arguments] ')' { z }] ':' c=block { + | 'class' a=NAME t=[type_params] b=['(' z=[arguments] ')' { z }] ':' c=block { _PyAST_ClassDef(a->v.Name.id, (b) ? ((expr_ty) b)->v.Call.args : NULL, (b) ? ((expr_ty) b)->v.Call.keywords : NULL, - c, NULL, EXTRA) } + c, NULL, t, EXTRA) } # Function definitions # -------------------- @@ -267,18 +266,18 @@ function_def[stmt_ty]: function_def_raw[stmt_ty]: | invalid_def_raw - | 'def' n=NAME &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { _PyAST_FunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), - b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | ASYNC 'def' n=NAME &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) } + | 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { CHECK_VERSION( stmt_ty, 5, "Async functions are", _PyAST_AsyncFunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), - b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) + b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) ) } # Function parameters @@ -384,7 +383,7 @@ for_stmt[stmt_ty]: | invalid_for_stmt | 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { _PyAST_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | ASYNC 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { + | 'async' 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { CHECK_VERSION(stmt_ty, 5, "Async for loops are", _PyAST_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) } | invalid_for_target @@ -393,13 +392,13 @@ for_stmt[stmt_ty]: with_stmt[stmt_ty]: | invalid_with_stmt_indent - | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { - CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NULL, EXTRA)) } + | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' tc=[TYPE_COMMENT] b=block { + CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | ASYNC 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { + | 'async' 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { CHECK_VERSION(stmt_ty, 5, "Async with statements are", _PyAST_AsyncWith(a, b, NULL, EXTRA)) } - | ASYNC 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { + | 'async' 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { CHECK_VERSION(stmt_ty, 5, "Async with statements are", _PyAST_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } | invalid_with_stmt @@ -628,6 +627,39 @@ keyword_patterns[asdl_seq*]: keyword_pattern[KeyPatternPair*]: | arg=NAME '=' value=pattern { _PyPegen_key_pattern_pair(p, arg, value) } +# Type statement +# --------------- + +type_alias[stmt_ty]: + | "type" n=NAME t=[type_params] '=' b=expression { + CHECK_VERSION(stmt_ty, 12, "Type statement is", + _PyAST_TypeAlias(CHECK(expr_ty, _PyPegen_set_expr_context(p, n, Store)), t, b, EXTRA)) } + +# Type parameter declaration +# -------------------------- + +type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { + CHECK_VERSION(asdl_type_param_seq *, 12, "Type parameter lists are", t) } + +type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } + +type_param[type_param_ty] (memo): + | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) } + | '*' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with TypeVarTuple" + : "cannot use bound with TypeVarTuple") + } + | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) } + | '**' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with ParamSpec" + : "cannot use bound with ParamSpec") + } + | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) } + +type_param_bound[expr_ty]: ':' e=expression { e } + # EXPRESSIONS # ----------- @@ -780,7 +812,7 @@ power[expr_ty]: # Primary elements are things like "obj.something.something", "obj[something]", "obj(something)", "obj" ... await_primary[expr_ty] (memo): - | AWAIT a=primary { CHECK_VERSION(expr_ty, 5, "Await expressions are", _PyAST_Await(a, EXTRA)) } + | 'await' a=primary { CHECK_VERSION(expr_ty, 5, "Await expressions are", _PyAST_Await(a, EXTRA)) } | primary primary[expr_ty]: @@ -881,7 +913,7 @@ fstring_middle[expr_ty]: | fstring_replacement_field | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } fstring_replacement_field[expr_ty]: - | '{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' { + | '{' a=(yield_expr | star_expressions) debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' { _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) } | invalid_replacement_field fstring_conversion[ResultTokenWithMetadata*]: @@ -889,7 +921,7 @@ fstring_conversion[ResultTokenWithMetadata*]: fstring_full_format_spec[ResultTokenWithMetadata*]: | colon=':' spec=fstring_format_spec* { _PyPegen_setup_full_format_spec(p, colon, (asdl_expr_seq *) spec, EXTRA) } fstring_format_spec[expr_ty]: - | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } + | t=FSTRING_MIDDLE { _PyPegen_decoded_constant_from_token(p, t) } | fstring_replacement_field fstring[expr_ty]: | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, a, (asdl_expr_seq*)b, c) } @@ -932,7 +964,7 @@ for_if_clauses[asdl_comprehension_seq*]: | a[asdl_comprehension_seq*]=for_if_clause+ { a } for_if_clause[comprehension_ty]: - | ASYNC 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { + | 'async' 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) } | 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { _PyAST_comprehension(a, b, c, 0, p->arena) } @@ -1097,7 +1129,8 @@ func_type_comment[Token*]: # From here on, there are rules for invalid syntax with specialised error messages invalid_arguments: - | a=args ',' '*' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable argument unpacking follows keyword argument unpacking") } + | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' b='*' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "iterable argument unpacking follows keyword argument unpacking") } | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { @@ -1136,7 +1169,7 @@ invalid_expression: _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } - | a='lambda' [lambda_params] b=':' &(FSTRING_MIDDLE | fstring_replacement_field) { + | a='lambda' [lambda_params] b=':' &FSTRING_MIDDLE { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "f-string: lambda expressions are not allowed without parentheses") } invalid_named_expression(memo): @@ -1250,7 +1283,7 @@ invalid_with_item: RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } invalid_for_target: - | ASYNC? 'for' a=star_expressions { + | 'async'? 'for' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) } invalid_group: @@ -1259,20 +1292,24 @@ invalid_group: | '(' a='**' expression ')' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use double starred expression here") } invalid_import: - | a='import' dotted_name 'from' dotted_name { + | a='import' ','.dotted_name+ 'from' dotted_name { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "Did you mean to use 'from ... import ...' instead?") } invalid_import_from_targets: | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } +invalid_compound_stmt: + | a='elif' named_expression ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "'elif' must match an if-statement here") } + | a='else' ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "'else' must match a valid statement here") } + invalid_with_stmt: - | [ASYNC] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } - | [ASYNC] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } invalid_with_stmt_indent: - | [ASYNC] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT { + | ['async'] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) } - | [ASYNC] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT { + | ['async'] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) } invalid_try_stmt: @@ -1333,15 +1370,15 @@ invalid_while_stmt: | a='while' named_expression ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'while' statement on line %d", a->lineno) } invalid_for_stmt: - | [ASYNC] 'for' star_targets 'in' star_expressions NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } - | [ASYNC] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT { + | ['async'] 'for' star_targets 'in' star_expressions NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | ['async'] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'for' statement on line %d", a->lineno) } invalid_def_raw: - | [ASYNC] a='def' NAME '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { + | ['async'] a='def' NAME [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) } invalid_class_def_raw: - | 'class' NAME ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } - | a='class' NAME ['(' [arguments] ')'] ':' NEWLINE !INDENT { + | 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | a='class' NAME [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after class definition on line %d", a->lineno) } invalid_double_starred_kvpairs: diff --git a/Include/Python.h b/Include/Python.h index 52a7aac6ba6cb6..196751c3201e62 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -5,42 +5,56 @@ #ifndef Py_PYTHON_H #define Py_PYTHON_H -// Since this is a "meta-include" file, no #ifdef __cplusplus / extern "C" { +// Since this is a "meta-include" file, "#ifdef __cplusplus / extern "C" {" +// is not needed. + // Include Python header files #include "patchlevel.h" #include "pyconfig.h" #include "pymacconfig.h" -#if defined(__sgi) && !defined(_SGI_MP_SOURCE) -# define _SGI_MP_SOURCE + +// Include standard header files +#include // assert() +#include // uintptr_t +#include // INT_MAX +#include // HUGE_VAL +#include // va_list +#include // wchar_t +#ifdef HAVE_SYS_TYPES_H +# include // ssize_t #endif -// stdlib.h, stdio.h, errno.h and string.h headers are not used by Python -// headers, but kept for backward compatibility. They are excluded from the -// limited C API of Python 3.11. +// , , and headers are no longer used +// by Python, but kept for the backward compatibility of existing third party C +// extensions. They are not included by limited C API version 3.11 and newer. +// +// The and headers are not included by limited C API +// version 3.13 and newer. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# include -# include // FILE* # include // errno +# include // FILE* +# include // getenv() # include // memcpy() #endif -#ifndef MS_WINDOWS -# include -#endif -#ifdef HAVE_STDDEF_H -# include // size_t +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030d0000 +# include // tolower() +# ifndef MS_WINDOWS +# include // close() +# endif #endif -#include // assert() -#include // wchar_t +// Include Python header files #include "pyport.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" #include "pytypedefs.h" #include "pybuffer.h" +#include "pystats.h" +#include "pyatomic.h" #include "object.h" #include "objimpl.h" #include "typeslots.h" @@ -83,7 +97,6 @@ #include "weakrefobject.h" #include "structseq.h" #include "cpython/picklebufobject.h" -#include "cpython/pytime.h" #include "codecs.h" #include "pyerrors.h" #include "pythread.h" @@ -104,6 +117,7 @@ #include "pystrcmp.h" #include "fileutils.h" #include "cpython/pyfpe.h" -#include "tracemalloc.h" +#include "cpython/tracemalloc.h" +#include "cpython/optimizer.h" #endif /* !Py_PYTHON_H */ diff --git a/Include/abstract.h b/Include/abstract.h index 064b0300b51ea2..bd12a54963c13f 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -50,6 +50,25 @@ extern "C" { This function always succeeds. */ + +/* Implemented elsewhere: + + int PyObject_HasAttrStringWithError(PyObject *o, const char *attr_name); + + Returns 1 if object 'o' has the attribute attr_name, and 0 otherwise. + This is equivalent to the Python expression: hasattr(o,attr_name). + Returns -1 on failure. */ + + +/* Implemented elsewhere: + + int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name); + + Returns 1 if o has the attribute attr_name, and 0 otherwise. + This is equivalent to the Python expression: hasattr(o,attr_name). + Returns -1 on failure. */ + + /* Implemented elsewhere: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name); @@ -60,6 +79,38 @@ extern "C" { This is the equivalent of the Python expression: o.attr_name. */ +/* Implemented elsewhere: + + int PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result); + + Variant of PyObject_GetAttr() which doesn't raise AttributeError + if the attribute is not found. + + If the attribute is found, return 1 and set *result to a new strong + reference to the attribute. + If the attribute is not found, return 0 and set *result to NULL; + the AttributeError is silenced. + If an error other than AttributeError is raised, return -1 and + set *result to NULL. +*/ + + +/* Implemented elsewhere: + + int PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result); + + Variant of PyObject_GetAttrString() which doesn't raise AttributeError + if the attribute is not found. + + If the attribute is found, return 1 and set *result to a new strong + reference to the attribute. + If the attribute is not found, return 0 and set *result to NULL; + the AttributeError is silenced. + If an error other than AttributeError is raised, return -1 and + set *result to NULL. +*/ + + /* Implemented elsewhere: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v); @@ -80,7 +131,7 @@ extern "C" { This is the equivalent of the Python statement o.attr_name=v. */ -/* Implemented as a macro: +/* Implemented elsewhere: int PyObject_DelAttrString(PyObject *o, const char *attr_name); @@ -88,17 +139,15 @@ extern "C" { -1 on failure. This is the equivalent of the Python statement: del o.attr_name. */ -#define PyObject_DelAttrString(O, A) PyObject_SetAttrString((O), (A), NULL) -/* Implemented as a macro: +/* Implemented elsewhere: int PyObject_DelAttr(PyObject *o, PyObject *attr_name); Delete attribute named attr_name, for object o. Returns -1 on failure. This is the equivalent of the Python statement: del o.attr_name. */ -#define PyObject_DelAttr(O, A) PyObject_SetAttr((O), (A), NULL) /* Implemented elsewhere: @@ -135,12 +184,6 @@ extern "C" { This function always succeeds. */ -#ifdef PY_SSIZE_T_CLEAN -# define PyObject_CallFunction _PyObject_CallFunction_SizeT -# define PyObject_CallMethod _PyObject_CallMethod_SizeT -#endif - - #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 /* Call a callable Python object without any arguments */ PyAPI_FUNC(PyObject *) PyObject_CallNoArgs(PyObject *func); @@ -195,15 +238,6 @@ PyAPI_FUNC(PyObject *) PyObject_CallMethod(PyObject *obj, const char *name, const char *format, ...); -PyAPI_FUNC(PyObject *) _PyObject_CallFunction_SizeT(PyObject *callable, - const char *format, - ...); - -PyAPI_FUNC(PyObject *) _PyObject_CallMethod_SizeT(PyObject *obj, - const char *name, - const char *format, - ...); - /* Call a callable Python object 'callable' with a variable number of C arguments. The C arguments are provided as PyObject* values, terminated by a NULL. @@ -335,55 +369,6 @@ PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, const char *key); PyAPI_FUNC(int) PyObject_DelItem(PyObject *o, PyObject *key); -/* === Old Buffer API ============================================ */ - -/* FIXME: usage of these should all be replaced in Python itself - but for backwards compatibility we will implement them. - Their usage without a corresponding "unlock" mechanism - may create issues (but they would already be there). */ - -/* Takes an arbitrary object which must support the (character, single segment) - buffer interface and returns a pointer to a read-only memory location - usable as character based input for subsequent processing. - - Return 0 on success. buffer and buffer_len are only set in case no error - occurs. Otherwise, -1 is returned and an exception set. */ -Py_DEPRECATED(3.0) -PyAPI_FUNC(int) PyObject_AsCharBuffer(PyObject *obj, - const char **buffer, - Py_ssize_t *buffer_len); - -/* Checks whether an arbitrary object supports the (character, single segment) - buffer interface. - - Returns 1 on success, 0 on failure. */ -Py_DEPRECATED(3.0) PyAPI_FUNC(int) PyObject_CheckReadBuffer(PyObject *obj); - -/* Same as PyObject_AsCharBuffer() except that this API expects (readable, - single segment) buffer interface and returns a pointer to a read-only memory - location which can contain arbitrary data. - - 0 is returned on success. buffer and buffer_len are only set in case no - error occurs. Otherwise, -1 is returned and an exception set. */ -Py_DEPRECATED(3.0) -PyAPI_FUNC(int) PyObject_AsReadBuffer(PyObject *obj, - const void **buffer, - Py_ssize_t *buffer_len); - -/* Takes an arbitrary object which must support the (writable, single segment) - buffer interface and returns a pointer to a writable memory location in - buffer of size 'buffer_len'. - - Return 0 on success. buffer and buffer_len are only set in case no error - occurs. Otherwise, -1 is returned and an exception set. */ -Py_DEPRECATED(3.0) -PyAPI_FUNC(int) PyObject_AsWriteBuffer(PyObject *obj, - void **buffer, - Py_ssize_t *buffer_len); - - -/* === New Buffer API ============================================ */ - /* Takes an arbitrary object and returns the result of calling obj.__format__(format_spec). */ PyAPI_FUNC(PyObject *) PyObject_Format(PyObject *obj, @@ -855,6 +840,18 @@ PyAPI_FUNC(int) PyMapping_HasKeyString(PyObject *o, const char *key); This function always succeeds. */ PyAPI_FUNC(int) PyMapping_HasKey(PyObject *o, PyObject *key); +/* Return 1 if the mapping object has the key 'key', and 0 otherwise. + This is equivalent to the Python expression: key in o. + On failure, return -1. */ + +PyAPI_FUNC(int) PyMapping_HasKeyWithError(PyObject *o, PyObject *key); + +/* Return 1 if the mapping object has the key 'key', and 0 otherwise. + This is equivalent to the Python expression: key in o. + On failure, return -1. */ + +PyAPI_FUNC(int) PyMapping_HasKeyStringWithError(PyObject *o, const char *key); + /* On success, return a list or tuple of the keys in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Keys(PyObject *o); @@ -874,6 +871,21 @@ PyAPI_FUNC(PyObject *) PyMapping_Items(PyObject *o); PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o, const char *key); +/* Variants of PyObject_GetItem() and PyMapping_GetItemString() which don't + raise KeyError if the key is not found. + + If the key is found, return 1 and set *result to a new strong + reference to the corresponding value. + If the key is not found, return 0 and set *result to NULL; + the KeyError is silenced. + If an error other than KeyError is raised, return -1 and + set *result to NULL. +*/ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PyMapping_GetOptionalItem(PyObject *, PyObject *, PyObject **); +PyAPI_FUNC(int) PyMapping_GetOptionalItemString(PyObject *, const char *, PyObject **); +#endif + /* Map the string 'key' to the value 'v' in the mapping 'o'. Returns -1 on failure. diff --git a/Include/boolobject.h b/Include/boolobject.h index 976fa35201d035..19aef5b1b87c6a 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -7,7 +7,7 @@ extern "C" { #endif -PyAPI_DATA(PyTypeObject) PyBool_Type; +// PyBool_Type is declared by object.h #define PyBool_Check(x) Py_IS_TYPE((x), &PyBool_Type) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index ee448cd02bdab3..c5a24195be6bc3 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -1,5 +1,4 @@ - -/* Bytes object interface */ +// Bytes object interface #ifndef Py_BYTESOBJECT_H #define Py_BYTESOBJECT_H @@ -7,8 +6,6 @@ extern "C" { #endif -#include // va_list - /* Type PyBytesObject represents a byte string. An extra zero byte is reserved at the end to ensure it is zero-terminated, but a size is diff --git a/Include/ceval.h b/Include/ceval.h index ad4d909d6f2b14..9885bdb7febc21 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -17,27 +17,6 @@ PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyObject *co, PyObject *const *defs, int defc, PyObject *kwdefs, PyObject *closure); -/* PyEval_CallObjectWithKeywords(), PyEval_CallObject(), PyEval_CallFunction - * and PyEval_CallMethod are deprecated. Since they are officially part of the - * stable ABI (PEP 384), they must be kept for backward compatibility. - * PyObject_Call(), PyObject_CallFunction() and PyObject_CallMethod() are - * recommended to call a callable object. - */ - -Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyEval_CallObjectWithKeywords( - PyObject *callable, - PyObject *args, - PyObject *kwargs); - -/* Deprecated since PyEval_CallObjectWithKeywords is deprecated */ -#define PyEval_CallObject(callable, arg) \ - PyEval_CallObjectWithKeywords((callable), (arg), _PyObject_CAST(_Py_NULL)) - -Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyEval_CallFunction( - PyObject *callable, const char *format, ...); -Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyEval_CallMethod( - PyObject *obj, const char *name, const char *format, ...); - PyAPI_FUNC(PyObject *) PyEval_GetBuiltins(void); PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); @@ -128,14 +107,6 @@ PyAPI_FUNC(PyObject *) PyEval_EvalFrameEx(PyFrameObject *f, int exc); PyAPI_FUNC(PyThreadState *) PyEval_SaveThread(void); PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState *); -Py_DEPRECATED(3.9) PyAPI_FUNC(int) PyEval_ThreadsInitialized(void); -Py_DEPRECATED(3.9) PyAPI_FUNC(void) PyEval_InitThreads(void); -/* PyEval_AcquireLock() and PyEval_ReleaseLock() are part of stable ABI. - * They will be removed from this header file in the future version. - * But they will be remained in ABI until Python 4.0. - */ -Py_DEPRECATED(3.2) PyAPI_FUNC(void) PyEval_AcquireLock(void); -Py_DEPRECATED(3.2) PyAPI_FUNC(void) PyEval_ReleaseLock(void); PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate); PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); diff --git a/Include/codecs.h b/Include/codecs.h index 37ecfb4ab757b4..512a3c723eca18 100644 --- a/Include/codecs.h +++ b/Include/codecs.h @@ -35,34 +35,6 @@ PyAPI_FUNC(int) PyCodec_Unregister( PyObject *search_function ); -/* Codec registry lookup API. - - Looks up the given encoding and returns a CodecInfo object with - function attributes which implement the different aspects of - processing the encoding. - - The encoding string is looked up converted to all lower-case - characters. This makes encodings looked up through this mechanism - effectively case-insensitive. - - If no codec is found, a KeyError is set and NULL returned. - - As side effect, this tries to load the encodings package, if not - yet done. This is part of the lazy load strategy for the encodings - package. - - */ - -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyCodec_Lookup( - const char *encoding - ); - -PyAPI_FUNC(int) _PyCodec_Forget( - const char *encoding - ); -#endif - /* Codec registry encoding check API. Returns 1/0 depending on whether there is a registered codec for @@ -106,102 +78,58 @@ PyAPI_FUNC(PyObject *) PyCodec_Decode( const char *errors ); -#ifndef Py_LIMITED_API -/* Text codec specific encoding and decoding API. - - Checks the encoding against a list of codecs which do not - implement a str<->bytes encoding before attempting the - operation. +// --- Codec Lookup APIs -------------------------------------------------- - Please note that these APIs are internal and should not - be used in Python C extensions. - - XXX (ncoghlan): should we make these, or something like them, public - in Python 3.5+? +/* Codec registry lookup API. - */ -PyAPI_FUNC(PyObject *) _PyCodec_LookupTextEncoding( - const char *encoding, - const char *alternate_command - ); + Looks up the given encoding and returns a CodecInfo object with + function attributes which implement the different aspects of + processing the encoding. -PyAPI_FUNC(PyObject *) _PyCodec_EncodeText( - PyObject *object, - const char *encoding, - const char *errors - ); + The encoding string is looked up converted to all lower-case + characters. This makes encodings looked up through this mechanism + effectively case-insensitive. -PyAPI_FUNC(PyObject *) _PyCodec_DecodeText( - PyObject *object, - const char *encoding, - const char *errors - ); + If no codec is found, a KeyError is set and NULL returned. -/* These two aren't actually text encoding specific, but _io.TextIOWrapper - * is the only current API consumer. + As side effect, this tries to load the encodings package, if not + yet done. This is part of the lazy load strategy for the encodings + package. */ -PyAPI_FUNC(PyObject *) _PyCodecInfo_GetIncrementalDecoder( - PyObject *codec_info, - const char *errors - ); - -PyAPI_FUNC(PyObject *) _PyCodecInfo_GetIncrementalEncoder( - PyObject *codec_info, - const char *errors - ); -#endif - - - -/* --- Codec Lookup APIs -------------------------------------------------- - - All APIs return a codec object with incremented refcount and are - based on _PyCodec_Lookup(). The same comments w/r to the encoding - name also apply to these APIs. - -*/ /* Get an encoder function for the given encoding. */ -PyAPI_FUNC(PyObject *) PyCodec_Encoder( - const char *encoding - ); +PyAPI_FUNC(PyObject *) PyCodec_Encoder(const char *encoding); /* Get a decoder function for the given encoding. */ -PyAPI_FUNC(PyObject *) PyCodec_Decoder( - const char *encoding - ); +PyAPI_FUNC(PyObject *) PyCodec_Decoder(const char *encoding); /* Get an IncrementalEncoder object for the given encoding. */ PyAPI_FUNC(PyObject *) PyCodec_IncrementalEncoder( - const char *encoding, - const char *errors - ); + const char *encoding, + const char *errors); /* Get an IncrementalDecoder object function for the given encoding. */ PyAPI_FUNC(PyObject *) PyCodec_IncrementalDecoder( - const char *encoding, - const char *errors - ); + const char *encoding, + const char *errors); /* Get a StreamReader factory function for the given encoding. */ PyAPI_FUNC(PyObject *) PyCodec_StreamReader( - const char *encoding, - PyObject *stream, - const char *errors - ); + const char *encoding, + PyObject *stream, + const char *errors); /* Get a StreamWriter factory function for the given encoding. */ PyAPI_FUNC(PyObject *) PyCodec_StreamWriter( - const char *encoding, - PyObject *stream, - const char *errors - ); + const char *encoding, + PyObject *stream, + const char *errors); /* Unicode encoding error handling callback registry API */ diff --git a/Include/compile.h b/Include/compile.h index 3c5acd7209f763..52d0bc76c9fca4 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -10,9 +10,6 @@ extern "C" { #define Py_eval_input 258 #define Py_func_type_input 345 -/* This doesn't need to match anything */ -#define Py_fstring_input 800 - #ifndef Py_LIMITED_API # define Py_CPYTHON_COMPILE_H # include "cpython/compile.h" diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 3b27aab2fc4798..4e7b7a46703a6d 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -4,9 +4,12 @@ /* === Object Protocol ================================================== */ -#ifdef PY_SSIZE_T_CLEAN -# define _PyObject_CallMethodId _PyObject_CallMethodId_SizeT -#endif +/* Like PyObject_CallMethod(), but expect a _Py_Identifier* + as the method name. */ +PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( + PyObject *obj, + _Py_Identifier *name, + const char *format, ...); /* Convert keyword arguments from the FASTCALL (stack: C array, kwnames: tuple) format to a Python dictionary ("kwargs" dict). @@ -18,51 +21,25 @@ Duplicate keys are merged using the last value. If duplicate keys must raise an exception, the caller is responsible to implement an explicit keys on kwnames. */ -PyAPI_FUNC(PyObject *) _PyStack_AsDict( - PyObject *const *values, - PyObject *kwnames); - -/* Suggested size (number of positional arguments) for arrays of PyObject* - allocated on a C stack to avoid allocating memory on the heap memory. Such - array is used to pass positional arguments to call functions of the - PyObject_Vectorcall() family. - - The size is chosen to not abuse the C stack and so limit the risk of stack - overflow. The size is also chosen to allow using the small stack for most - function calls of the Python standard library. On 64-bit CPU, it allocates - 40 bytes on the stack. */ -#define _PY_FASTCALL_SMALL_STACK 5 - -PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult( - PyThreadState *tstate, - PyObject *callable, - PyObject *result, - const char *where); +PyAPI_FUNC(PyObject*) _PyStack_AsDict(PyObject *const *values, PyObject *kwnames); -/* === Vectorcall protocol (PEP 590) ============================= */ -/* Call callable using tp_call. Arguments are like PyObject_Vectorcall() - or PyObject_FastCallDict() (both forms are supported), - except that nargs is plainly the number of arguments without flags. */ -PyAPI_FUNC(PyObject *) _PyObject_MakeTpCall( - PyThreadState *tstate, - PyObject *callable, - PyObject *const *args, Py_ssize_t nargs, - PyObject *keywords); +/* === Vectorcall protocol (PEP 590) ============================= */ // PyVectorcall_NARGS() is exported as a function for the stable ABI. // Here (when we are not using the stable ABI), the name is overridden to // call a static inline function for best performance. -#define PyVectorcall_NARGS(n) _PyVectorcall_NARGS(n) static inline Py_ssize_t _PyVectorcall_NARGS(size_t n) { return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET; } +#define PyVectorcall_NARGS(n) _PyVectorcall_NARGS(n) PyAPI_FUNC(vectorcallfunc) PyVectorcall_Function(PyObject *callable); -// Backwards compatibility aliases for API that was provisional in Python 3.8 +// Backwards compatibility aliases (PEP 590) for API that was provisional +// in Python 3.8 #define _PyObject_Vectorcall PyObject_Vectorcall #define _PyObject_VectorcallMethod PyObject_VectorcallMethod #define _PyObject_FastCallDict PyObject_VectorcallDict @@ -79,12 +56,6 @@ PyAPI_FUNC(PyObject *) PyObject_VectorcallDict( size_t nargsf, PyObject *kwargs); -// Same as PyObject_Vectorcall(), except without keyword arguments -PyAPI_FUNC(PyObject *) _PyObject_FastCall( - PyObject *func, - PyObject *const *args, - Py_ssize_t nargs); - PyAPI_FUNC(PyObject *) PyObject_CallOneArg(PyObject *func, PyObject *arg); static inline PyObject * @@ -103,56 +74,6 @@ PyObject_CallMethodOneArg(PyObject *self, PyObject *name, PyObject *arg) return PyObject_VectorcallMethod(name, args, nargsf, _Py_NULL); } -PyAPI_FUNC(PyObject *) _PyObject_CallMethod(PyObject *obj, - PyObject *name, - const char *format, ...); - -/* Like PyObject_CallMethod(), but expect a _Py_Identifier* - as the method name. */ -PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *obj, - _Py_Identifier *name, - const char *format, ...); - -PyAPI_FUNC(PyObject *) _PyObject_CallMethodId_SizeT(PyObject *obj, - _Py_Identifier *name, - const char *format, - ...); - -PyAPI_FUNC(PyObject *) _PyObject_CallMethodIdObjArgs( - PyObject *obj, - _Py_Identifier *name, - ...); - -static inline PyObject * -_PyObject_VectorcallMethodId( - _Py_Identifier *name, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return _Py_NULL; - } - return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); -} - -static inline PyObject * -_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) -{ - size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); -} - -static inline PyObject * -_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) -{ - PyObject *args[2] = {self, arg}; - size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - assert(arg != NULL); - return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); -} - -PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); - /* Guess the size of object 'o' using len(o) or o.__length_hint__(). If neither of those return a non-negative value, then return the default value. If one of the calls fails, this function returns -1. */ @@ -164,43 +85,3 @@ PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t); need to be corrected for a negative index. */ #define PySequence_ITEM(o, i)\ ( Py_TYPE(o)->tp_as_sequence->sq_item((o), (i)) ) - -#define PY_ITERSEARCH_COUNT 1 -#define PY_ITERSEARCH_INDEX 2 -#define PY_ITERSEARCH_CONTAINS 3 - -/* Iterate over seq. - - Result depends on the operation: - - PY_ITERSEARCH_COUNT: return # of times obj appears in seq; -1 if - error. - PY_ITERSEARCH_INDEX: return 0-based index of first occurrence of - obj in seq; set ValueError and return -1 if none found; - also return -1 on error. - PY_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on - error. */ -PyAPI_FUNC(Py_ssize_t) _PySequence_IterSearch(PyObject *seq, - PyObject *obj, int operation); - -/* === Mapping protocol ================================================= */ - -PyAPI_FUNC(int) _PyObject_RealIsInstance(PyObject *inst, PyObject *cls); - -PyAPI_FUNC(int) _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls); - -PyAPI_FUNC(char *const *) _PySequence_BytesToCharpArray(PyObject* self); - -PyAPI_FUNC(void) _Py_FreeCharPArray(char *const array[]); - -/* For internal use by buffer API functions */ -PyAPI_FUNC(void) _Py_add_one_to_index_F(int nd, Py_ssize_t *index, - const Py_ssize_t *shape); -PyAPI_FUNC(void) _Py_add_one_to_index_C(int nd, Py_ssize_t *index, - const Py_ssize_t *shape); - -/* Convert Python int to Py_ssize_t. Do nothing if the argument is None. */ -PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); - -/* Same as PyNumber_Index but can return an instance of a subclass of int. */ -PyAPI_FUNC(PyObject *) _PyNumber_Index(PyObject *o); diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index e982031c107de2..816823716e9a6f 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -15,18 +15,6 @@ typedef struct { } PyBytesObject; PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( - const char *format, - Py_ssize_t format_len, - PyObject *args, - int use_bytearray); -PyAPI_FUNC(PyObject*) _PyBytes_FromHex( - PyObject *string, - int use_bytearray); - -/* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */ -PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, const char **); /* Macros and static inline functions, trading safety for speed */ #define _PyBytes_CAST(op) \ @@ -43,87 +31,3 @@ static inline Py_ssize_t PyBytes_GET_SIZE(PyObject *op) { return Py_SIZE(self); } #define PyBytes_GET_SIZE(self) PyBytes_GET_SIZE(_PyObject_CAST(self)) - -/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, - x must be an iterable object. */ -PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x); - - -/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes, bytearray or NULL (when the small buffer is used) */ - PyObject *buffer; - - /* Number of allocated size. */ - Py_ssize_t allocated; - - /* Minimum number of allocated bytes, - incremented by _PyBytesWriter_Prepare() */ - Py_ssize_t min_size; - - /* If non-zero, use a bytearray instead of a bytes object for buffer. */ - int use_bytearray; - - /* If non-zero, overallocate the buffer (default: 0). - This flag must be zero if use_bytearray is non-zero. */ - int overallocate; - - /* Stack buffer */ - int use_small_buffer; - char small_buffer[512]; -} _PyBytesWriter; - -/* Initialize a bytes writer - - By default, the overallocation is disabled. Set the overallocate attribute - to control the allocation of the buffer. */ -PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); - -/* Get the buffer content and reset the writer. - Return a bytes object, or a bytearray object if use_bytearray is non-zero. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - void *str); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, - Py_ssize_t size); - -/* Ensure that the buffer is large enough to write *size* bytes. - Add size to the writer minimum size (min_size attribute). - - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Resize the buffer to make it larger. - The new buffer may be larger than size bytes because of overallocation. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. - - Note: size must be greater than the number of allocated bytes in the writer. - - This function doesn't use the writer minimum size (min_size attribute). - - See also _PyBytesWriter_Prepare(). - */ -PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Write bytes. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - void *str, - const void *bytes, - Py_ssize_t size); diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 0fbbee10c2edce..78f7405661662f 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -4,14 +4,9 @@ PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetProfileAllThreads(Py_tracefunc, PyObject *); -PyAPI_DATA(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetTraceAllThreads(Py_tracefunc, PyObject *); -PyAPI_FUNC(int) _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); -/* Helper to look up a builtin object */ -PyAPI_FUNC(PyObject *) _PyEval_GetBuiltin(PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *); /* Look at the current frame's (if any) code's co_flags, and turn on the corresponding compiler flags in cf->cf_flags. Return 1 if any flag was set, else return 0. */ @@ -19,9 +14,6 @@ PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf); PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc); -PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds); -PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void); - PyAPI_FUNC(Py_ssize_t) PyUnstable_Eval_RequestCodeExtraIndex(freefunc); // Old name -- remove when this API changes: _Py_DEPRECATED_EXTERNALLY(3.12) static inline Py_ssize_t diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 6bead361c79245..cf715c55a2b3b8 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -8,16 +8,21 @@ extern "C" { #endif - +/* Count of all local monitoring events */ +#define _PY_MONITORING_LOCAL_EVENTS 10 /* Count of all "real" monitoring events (not derived from other events) */ -#define PY_MONITORING_UNGROUPED_EVENTS 14 +#define _PY_MONITORING_UNGROUPED_EVENTS 15 /* Count of all monitoring events */ -#define PY_MONITORING_EVENTS 16 +#define _PY_MONITORING_EVENTS 17 + +/* Tables of which tools are active for each monitored event. */ +typedef struct _Py_LocalMonitors { + uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; +} _Py_LocalMonitors; -/* Table of which tools are active for each monitored event. */ -typedef struct _Py_Monitors { - uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS]; -} _Py_Monitors; +typedef struct _Py_GlobalMonitors { + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; +} _Py_GlobalMonitors; /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG @@ -76,14 +81,21 @@ typedef struct { int8_t line_delta; } _PyCoLineInstrumentationData; + +typedef struct { + int size; + int capacity; + struct _PyExecutorObject *executors[1]; +} _PyExecutorArray; + /* Main data structure used for instrumentation. * This is allocated when needed for instrumentation */ typedef struct { /* Monitoring specific to this code object */ - _Py_Monitors local_monitors; + _Py_LocalMonitors local_monitors; /* Monitoring that is active on this code object */ - _Py_Monitors active_monitors; + _Py_LocalMonitors active_monitors; /* The tools that are to be notified for events for the matching code unit */ uint8_t *tools; /* Information to support line events */ @@ -153,8 +165,9 @@ typedef struct { PyObject *co_qualname; /* unicode (qualname, for reference) */ \ PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ + _PyExecutorArray *co_executors; /* executors from optimizer */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ - uint64_t _co_instrumentation_version; /* current instrumentation version */ \ + uintptr_t _co_instrumentation_version; /* current instrumentation version */ \ _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index f5a62a8ec6dd0c..0d587505ef7f85 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -19,9 +19,10 @@ #define PyCF_TYPE_COMMENTS 0x1000 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 #define PyCF_ALLOW_INCOMPLETE_INPUT 0x4000 +#define PyCF_OPTIMIZED_AST (0x8000 | PyCF_ONLY_AST) #define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | \ - PyCF_ALLOW_INCOMPLETE_INPUT) + PyCF_ALLOW_INCOMPLETE_INPUT | PyCF_OPTIMIZED_AST) typedef struct { int cf_flags; /* bitmask of CO_xxx flags relevant to future */ diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index b7d7283ae88965..fbdc6a91fe895c 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -7,8 +7,7 @@ typedef struct { double imag; } Py_complex; -/* Operations on complex numbers from complexmodule.c */ - +// Operations on complex numbers. PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); @@ -17,6 +16,7 @@ PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); PyAPI_FUNC(double) _Py_c_abs(Py_complex); + /* Complex object interface */ /* @@ -31,14 +31,3 @@ typedef struct { PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex); PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *op); - -#ifdef Py_BUILD_CORE -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -extern int _PyComplex_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); -#endif // Py_BUILD_CORE diff --git a/Include/cpython/context.h b/Include/cpython/context.h index 9879fc7192ebb8..a3249fc29b082e 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -67,10 +67,6 @@ PyAPI_FUNC(PyObject *) PyContextVar_Set(PyObject *var, PyObject *value); PyAPI_FUNC(int) PyContextVar_Reset(PyObject *var, PyObject *token); -/* This method is exposed only for CPython tests. Don not use it. */ -PyAPI_FUNC(PyObject *) _PyContext_NewHamtForTests(void); - - #ifdef __cplusplus } #endif diff --git a/Include/cpython/descrobject.h b/Include/cpython/descrobject.h index e2ea1b9a2d3058..bbad8b59c225ab 100644 --- a/Include/cpython/descrobject.h +++ b/Include/cpython/descrobject.h @@ -57,8 +57,6 @@ typedef struct { void *d_wrapped; /* This can be any function pointer */ } PyWrapperDescrObject; -PyAPI_DATA(PyTypeObject) _PyMethodWrapper_Type; - PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, struct wrapperbase *, void *); PyAPI_FUNC(int) PyDescr_IsData(PyObject *); diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index ddada922020aa4..944965fb9e5351 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -33,21 +33,10 @@ typedef struct { } PyDictObject; PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); -PyAPI_FUNC(PyObject *) _PyDict_GetItemWithError(PyObject *dp, PyObject *key); -PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); + Py_hash_t hash); PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, - PyObject *item, Py_hash_t hash); -PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); -PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, - int (*predicate)(PyObject *value)); -PyAPI_FUNC(int) _PyDict_Next( - PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { @@ -58,35 +47,13 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { } #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) -PyAPI_FUNC(int) _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); -PyAPI_FUNC(int) _PyDict_ContainsId(PyObject *, _Py_Identifier *); -PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); -PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp); -PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); -PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); -PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *, PyObject *, PyObject *); -#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) - -/* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, - the first occurrence of a key wins, if override is 1, the last occurrence - of a key wins, if override is 2, a KeyError with conflicting key as - argument is raised. -*/ -PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); -PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); - -PyAPI_FUNC(int) _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); -PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out); - -/* _PyDictView */ +PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); -typedef struct { - PyObject_HEAD - PyDictObject *dv_dict; -} _PyDictViewObject; +PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); -PyAPI_FUNC(PyObject *) _PyDictView_New(PyObject *, PyTypeObject *); -PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); +PyAPI_FUNC(int) PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result); +PyAPI_FUNC(int) PyDict_PopString(PyObject *dict, const char *key, PyObject **result); +PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value); /* Dictionary watchers */ diff --git a/Include/cpython/fileobject.h b/Include/cpython/fileobject.h index b70ec318986d82..e2d89c522bdd13 100644 --- a/Include/cpython/fileobject.h +++ b/Include/cpython/fileobject.h @@ -3,7 +3,6 @@ #endif PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *); -PyAPI_FUNC(char *) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*); /* The std printer acts as a preliminary sys.stderr until the new io infrastructure is in place. */ @@ -15,5 +14,3 @@ typedef PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *); PyAPI_FUNC(PyObject *) PyFile_OpenCode(const char *utf8path); PyAPI_FUNC(PyObject *) PyFile_OpenCodeObject(PyObject *path); PyAPI_FUNC(int) PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction hook, void *userData); - -PyAPI_FUNC(int) _PyLong_FileDescriptor_Converter(PyObject *, void *); diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 6f3efe36ede5d8..4e19535c656f2c 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -4,8 +4,6 @@ # error "this header file must not be included directly" #endif -struct _PyInterpreterFrame; - /* Standard object interface */ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *, @@ -29,18 +27,3 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); - -/* The following functions are for use by debuggers and other tools - * implementing custom frame evaluators with PEP 523. */ - -/* Returns the code object of the frame (strong reference). - * Does not raise an exception. */ -PyAPI_FUNC(PyCodeObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); - -/* Returns a byte ofsset into the last executed instruction. - * Does not raise an exception. */ -PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); - -/* Returns the currently executing line number, or -1 if there is no line number. - * Does not raise an exception. */ -PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index c716330cc3fbab..de2013323d2c72 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -41,6 +41,7 @@ typedef struct { PyObject *func_weakreflist; /* List of weak references */ PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_typeparams; /* Tuple of active type variables or NULL */ vectorcallfunc vectorcall; /* Version number for use by specializer. * Can set to non-zero when we want to specialize. @@ -78,12 +79,6 @@ PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *); PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( - PyObject *func, - PyObject *const *stack, - size_t nargsf, - PyObject *kwnames); - #define _PyFunction_CAST(func) \ (assert(PyFunction_Check(func)), _Py_CAST(PyFunctionObject*, func)) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index 7856481b5db300..49e46c277d75ae 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -41,9 +41,6 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, PyObject *name, PyObject *qualname); -PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); -PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); -PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self); PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); @@ -54,7 +51,6 @@ typedef struct { } PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; -PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type; #define PyCoro_CheckExact(op) Py_IS_TYPE((op), &PyCoro_Type) PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, @@ -69,8 +65,6 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; -PyAPI_DATA(PyTypeObject) _PyAsyncGenWrappedValue_Type; -PyAPI_DATA(PyTypeObject) _PyAsyncGenAThrow_Type; PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *name, PyObject *qualname); diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 2bca4ade4c4f2c..7daf0b84fcf71b 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -4,23 +4,6 @@ PyMODINIT_FUNC PyInit__imp(void); -PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); - -PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(_Py_Identifier *name); -PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); -PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); - -PyAPI_FUNC(void) _PyImport_AcquireLock(PyInterpreterState *interp); -PyAPI_FUNC(int) _PyImport_ReleaseLock(PyInterpreterState *interp); - -PyAPI_FUNC(int) _PyImport_FixupBuiltin( - PyObject *mod, - const char *name, /* UTF-8 encoded string */ - PyObject *modules - ); -PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, - PyObject *, PyObject *); - struct _inittab { const char *name; /* ASCII encoded string */ PyObject* (*initfunc)(void); @@ -34,13 +17,9 @@ struct _frozen { const unsigned char *code; int size; int is_package; - PyObject *(*get_code)(void); }; /* Embedding apps may change this pointer to point to their favorite collection of frozen modules: */ PyAPI_DATA(const struct _frozen *) PyImport_FrozenModules; - -PyAPI_DATA(PyObject *) _PyImport_GetModuleAttr(PyObject *, PyObject *); -PyAPI_DATA(PyObject *) _PyImport_GetModuleAttrString(const char *, const char *); diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index efae2409b50069..87c059c521cbc9 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -25,7 +25,6 @@ PyAPI_FUNC(PyStatus) PyStatus_Exit(int exitcode); PyAPI_FUNC(int) PyStatus_IsError(PyStatus err); PyAPI_FUNC(int) PyStatus_IsExit(PyStatus err); PyAPI_FUNC(int) PyStatus_Exception(PyStatus err); -PyAPI_FUNC(PyObject *) _PyErr_SetFromPyStatus(PyStatus status); /* --- PyWideStringList ------------------------------------------------ */ @@ -181,6 +180,8 @@ typedef struct PyConfig { int safe_path; int int_max_str_digits; + int cpu_count; + /* --- Path configuration inputs ------------ */ int pathconfig_warnings; wchar_t *program_name; @@ -205,6 +206,9 @@ typedef struct PyConfig { wchar_t *run_module; wchar_t *run_filename; + /* --- Set by Py_Main() -------------------------- */ + wchar_t *sys_path_0; + /* --- Private fields ---------------------------- */ // Install importlib? If equals to 0, importlib is not initialized at all. @@ -216,6 +220,17 @@ typedef struct PyConfig { // If non-zero, we believe we're running from a source tree. int _is_python_build; + +#ifdef Py_STATS + // If non-zero, turns on statistics gathering. + int _pystats; +#endif + +#ifdef Py_DEBUG + // If not empty, import a non-__main__ module before site.py is executed. + // PYTHON_PRESITE=package.module or -X presite=package.module + wchar_t *run_presite; +#endif } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); @@ -242,41 +257,6 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, Py_ssize_t length, wchar_t **items); -/* --- PyInterpreterConfig ------------------------------------ */ - -typedef struct { - // XXX "allow_object_sharing"? "own_objects"? - int use_main_obmalloc; - int allow_fork; - int allow_exec; - int allow_threads; - int allow_daemon_threads; - int check_multi_interp_extensions; - int own_gil; -} PyInterpreterConfig; - -#define _PyInterpreterConfig_INIT \ - { \ - .use_main_obmalloc = 0, \ - .allow_fork = 0, \ - .allow_exec = 0, \ - .allow_threads = 1, \ - .allow_daemon_threads = 0, \ - .check_multi_interp_extensions = 1, \ - .own_gil = 1, \ - } - -#define _PyInterpreterConfig_LEGACY_INIT \ - { \ - .use_main_obmalloc = 1, \ - .allow_fork = 1, \ - .allow_exec = 1, \ - .allow_threads = 1, \ - .allow_daemon_threads = 1, \ - .check_multi_interp_extensions = 0, \ - .own_gil = 0, \ - } - /* --- Helper functions --------------------------------------- */ /* Get the original command line arguments, before Python modified them. diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h new file mode 100644 index 00000000000000..4ab9ad5d315f80 --- /dev/null +++ b/Include/cpython/interpreteridobject.h @@ -0,0 +1,11 @@ +#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H +# error "this header file must not be included directly" +#endif + +/* Interpreter ID Object */ + +PyAPI_DATA(PyTypeObject) PyInterpreterID_Type; + +PyAPI_FUNC(PyObject *) PyInterpreterID_New(int64_t); +PyAPI_FUNC(PyObject *) PyInterpreterState_GetIDObject(PyInterpreterState *); +PyAPI_FUNC(PyInterpreterState *) PyInterpreterID_LookUp(PyObject *); diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index 8fa82122d8d248..8ade1b164681f9 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -21,9 +21,6 @@ typedef struct { Py_ssize_t allocated; } PyListObject; -PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); -PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); - /* Cast argument to PyListObject* type. */ #define _PyList_CAST(op) \ (assert(PyList_Check(op)), _Py_CAST(PyListObject*, (op))) @@ -41,7 +38,12 @@ static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) { static inline void PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { PyListObject *list = _PyList_CAST(op); + assert(0 <= index); + assert(index < list->allocated); list->ob_item[index] = value; } #define PyList_SET_ITEM(op, index, value) \ PyList_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) + +PyAPI_FUNC(int) PyList_Extend(PyObject *self, PyObject *iterable); +PyAPI_FUNC(int) PyList_Clear(PyObject *self); diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index c4cf820da5e4f2..f5ccbb704e8bb8 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -89,13 +89,42 @@ struct _longobject { _PyLongValue long_value; }; -PyAPI_FUNC(PyLongObject *) _PyLong_New(Py_ssize_t); +PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); -/* Return a copy of src. */ -PyAPI_FUNC(PyObject *) _PyLong_Copy(PyLongObject *src); +// Return a copy of src. +PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src); -PyAPI_FUNC(PyLongObject *) -_PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits); +PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( + int negative, + Py_ssize_t digit_count, + digit *digits); + + +/* Inline some internals for speed. These should be in pycore_long.h + * if user code didn't need them inlined. */ + +#define _PyLong_SIGN_MASK 3 +#define _PyLong_NON_SIZE_BITS 3 + + +static inline int +_PyLong_IsCompact(const PyLongObject* op) { + assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); + return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS); +} + +#define PyUnstable_Long_IsCompact _PyLong_IsCompact + +static inline Py_ssize_t +_PyLong_CompactValue(const PyLongObject *op) +{ + assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); + assert(PyUnstable_Long_IsCompact(op)); + Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + return sign * (Py_ssize_t)op->long_value.ob_digit[0]; +} + +#define PyUnstable_Long_CompactValue _PyLong_CompactValue #ifdef __cplusplus diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 1a73799d658fe0..fd1be29ed397d1 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -2,48 +2,16 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(int) _PyLong_AsInt(PyObject *); +PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); -PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); +PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); +PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); -/* _PyLong_Frexp returns a double x and an exponent e such that the - true value is approximately equal to x * 2**e. e is >= 0. x is - 0.0 if and only if the input is 0 (in which case, e and x are both - zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is - possible if the number of bits doesn't fit into a Py_ssize_t, sets - OverflowError and returns -1.0 for x, 0 for e. */ -PyAPI_FUNC(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); - -PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); -PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); - -/* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. - v must not be NULL, and must be a normalized long. - There are no error cases. -*/ +// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. +// v must not be NULL, and must be a normalized long. +// There are no error cases. PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); -/* _PyLong_NumBits. Return the number of bits needed to represent the - absolute value of a long. For example, this returns 1 for 1 and -1, 2 - for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. - v must not be NULL, and must be a normalized long. - (size_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. -*/ -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - -/* _PyLong_DivmodNear. Given integers a and b, compute the nearest - integer q to the exact quotient a / b, rounding to the nearest even integer - in the case of a tie. Return (q, r), where r = a - q*b. The remainder r - will satisfy abs(r) <= abs(b)/2, with equality possible only if q is - even. -*/ -PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); - /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. If n is 0, the integer is 0. Else: @@ -84,12 +52,5 @@ PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, int little_endian, int is_signed); -/* _PyLong_Format: Convert the long to a string object with given base, - appending a base prefix of 0[box] if base is 2, 8 or 16. */ -PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); - /* For use by the gcd function in mathmodule.c */ PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); - -PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t); -PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t); diff --git a/Include/cpython/memoryobject.h b/Include/cpython/memoryobject.h index 3837fa8c6ab5aa..961161b70f2058 100644 --- a/Include/cpython/memoryobject.h +++ b/Include/cpython/memoryobject.h @@ -2,8 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_DATA(PyTypeObject) _PyManagedBuffer_Type; - /* The structs are declared here so that macros can work, but they shouldn't be considered public. Don't access their fields directly, use the macros and functions instead! */ diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h deleted file mode 100644 index 88f34fe7513bf2..00000000000000 --- a/Include/cpython/modsupport.h +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef Py_CPYTHON_MODSUPPORT_H -# error "this header file must not be included directly" -#endif - -/* If PY_SSIZE_T_CLEAN is defined, each functions treats #-specifier - to mean Py_ssize_t */ -#ifdef PY_SSIZE_T_CLEAN -#define _Py_VaBuildStack _Py_VaBuildStack_SizeT -#else -PyAPI_FUNC(PyObject *) _Py_VaBuildValue_SizeT(const char *, va_list); -PyAPI_FUNC(PyObject **) _Py_VaBuildStack_SizeT( - PyObject **small_stack, - Py_ssize_t small_stack_len, - const char *format, - va_list va, - Py_ssize_t *p_nargs); -#endif - -PyAPI_FUNC(int) _PyArg_UnpackStack( - PyObject *const *args, - Py_ssize_t nargs, - const char *name, - Py_ssize_t min, - Py_ssize_t max, - ...); - -PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kwargs); -PyAPI_FUNC(int) _PyArg_NoKwnames(const char *funcname, PyObject *kwnames); -PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args); -#define _PyArg_NoKeywords(funcname, kwargs) \ - ((kwargs) == NULL || _PyArg_NoKeywords((funcname), (kwargs))) -#define _PyArg_NoKwnames(funcname, kwnames) \ - ((kwnames) == NULL || _PyArg_NoKwnames((funcname), (kwnames))) -#define _PyArg_NoPositional(funcname, args) \ - ((args) == NULL || _PyArg_NoPositional((funcname), (args))) - -#define _Py_ANY_VARARGS(n) ((n) == PY_SSIZE_T_MAX) - -PyAPI_FUNC(void) _PyArg_BadArgument(const char *, const char *, const char *, PyObject *); -PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, - Py_ssize_t, Py_ssize_t); -#define _PyArg_CheckPositional(funcname, nargs, min, max) \ - ((!_Py_ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ - || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) - -PyAPI_FUNC(PyObject **) _Py_VaBuildStack( - PyObject **small_stack, - Py_ssize_t small_stack_len, - const char *format, - va_list va, - Py_ssize_t *p_nargs); - -typedef struct _PyArg_Parser { - int initialized; - const char *format; - const char * const *keywords; - const char *fname; - const char *custom_msg; - int pos; /* number of positional-only arguments */ - int min; /* minimal number of arguments */ - int max; /* maximal number of positional arguments */ - PyObject *kwtuple; /* tuple of keyword parameter names */ - struct _PyArg_Parser *next; -} _PyArg_Parser; - -#ifdef PY_SSIZE_T_CLEAN -#define _PyArg_ParseTupleAndKeywordsFast _PyArg_ParseTupleAndKeywordsFast_SizeT -#define _PyArg_ParseStack _PyArg_ParseStack_SizeT -#define _PyArg_ParseStackAndKeywords _PyArg_ParseStackAndKeywords_SizeT -#define _PyArg_VaParseTupleAndKeywordsFast _PyArg_VaParseTupleAndKeywordsFast_SizeT -#endif - -PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, - struct _PyArg_Parser *, ...); -PyAPI_FUNC(int) _PyArg_ParseStack( - PyObject *const *args, - Py_ssize_t nargs, - const char *format, - ...); -PyAPI_FUNC(int) _PyArg_ParseStackAndKeywords( - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames, - struct _PyArg_Parser *, - ...); -PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywordsFast(PyObject *, PyObject *, - struct _PyArg_Parser *, va_list); -PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - PyObject **buf); - -PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - int vararg, PyObject **buf); - -#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ - (((minkw) == 0 && (kwargs) == NULL && (kwnames) == NULL && \ - (minpos) <= (nargs) && (nargs) <= (maxpos) && (args) != NULL) ? (args) : \ - _PyArg_UnpackKeywords((args), (nargs), (kwargs), (kwnames), (parser), \ - (minpos), (maxpos), (minkw), (buf))) - -PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(PyModuleDef*, int apiver); diff --git a/Include/cpython/object.h b/Include/cpython/object.h index d8eff691039d24..762e8a3b86ee1e 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -5,11 +5,6 @@ PyAPI_FUNC(void) _Py_NewReference(PyObject *op); PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op); -#ifdef Py_TRACE_REFS -/* Py_TRACE_REFS is such major surgery that we call external routines. */ -PyAPI_FUNC(void) _Py_ForgetReference(PyObject *); -#endif - #ifdef Py_REF_DEBUG /* These are useful as debugging aids when chasing down refleaks. */ PyAPI_FUNC(Py_ssize_t) _Py_GetGlobalRefTotal(void); @@ -56,6 +51,7 @@ typedef struct _Py_Identifier { #endif /* !Py_BUILD_CORE */ + typedef struct { /* Number implementations must check *both* arguments for proper type and implement the necessary conversions @@ -227,11 +223,11 @@ struct _typeobject { vectorcallfunc tp_vectorcall; /* bitset of which type-watchers care about this type */ - char tp_watched; + unsigned char tp_watched; }; /* This struct is used by the specializer - * It should should be treated as an opaque blob + * It should be treated as an opaque blob * by code other than the specializer and interpreter. */ struct _specialization_cache { // In order to avoid bloating the bytecode with lots of inline caches, the @@ -246,6 +242,7 @@ struct _specialization_cache { // *args nor **kwargs (as required by BINARY_SUBSCR_GETITEM): PyObject *getitem; uint32_t getitem_version; + PyObject *init; }; /* The *real* layout of a type object when allocated on the heap */ @@ -272,42 +269,16 @@ typedef struct _heaptypeobject { PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *); -PyAPI_FUNC(PyObject *) _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); -#ifndef Py_BUILD_CORE -// Backward compatibility for 3rd-party extensions -// that may be using the old name. -#define _PyObject_LookupSpecial _PyObject_LookupSpecialId -#endif -PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *); -PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); PyAPI_FUNC(void) _PyObject_Dump(PyObject *); -PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); - -PyAPI_FUNC(int) _PyObject_IsAbstract(PyObject *); -PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); -PyAPI_FUNC(int) _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); -/* Replacements of PyObject_GetAttr() and _PyObject_GetAttrId() which - don't raise AttributeError. - - Return 1 and set *result != NULL if an attribute is found. - Return 0 and set *result == NULL if an attribute is not found; - an AttributeError is silenced. - Return -1 and set *result == NULL if an error other than AttributeError - is raised. -*/ -PyAPI_FUNC(int) _PyObject_LookupAttr(PyObject *, PyObject *, PyObject **); -PyAPI_FUNC(int) _PyObject_LookupAttrId(PyObject *, _Py_Identifier *, PyObject **); -PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); -PyAPI_FUNC(PyObject *) _PyObject_NextNotImplemented(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); @@ -386,20 +357,6 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); #endif -PyAPI_DATA(PyTypeObject) _PyNone_Type; -PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; - -/* Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. - * Defined in object.c. - */ -PyAPI_DATA(int) _Py_SwappedOp[]; - -PyAPI_FUNC(void) -_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, - size_t sizeof_block); -PyAPI_FUNC(void) -_PyObject_DebugTypeStats(FILE *out); - /* Define a pair of assertion macros: _PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT(). @@ -448,21 +405,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed( int line, const char *function); -/* Check if an object is consistent. For example, ensure that the reference - counter is greater than or equal to 1, and ensure that ob_type is not NULL. - - Call _PyObject_AssertFailed() if the object is inconsistent. - - If check_content is zero, only check header fields: reduce the overhead. - - The function always return 1. The return value is just here to be able to - write: - - assert(_PyObject_CheckConsistency(obj, 1)); */ -PyAPI_FUNC(int) _PyObject_CheckConsistency( - PyObject *op, - int check_content); - /* Trashcan mechanism, thanks to Christian Tismer. @@ -523,7 +465,7 @@ PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); /* If "cond" is false, then _tstate remains NULL and the deallocator \ * is run normally without involving the trashcan */ \ if (cond) { \ - _tstate = _PyThreadState_UncheckedGet(); \ + _tstate = PyThreadState_GetUnchecked(); \ if (_PyTrash_begin(_tstate, _PyObject_CAST(op))) { \ break; \ } \ @@ -539,24 +481,11 @@ PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); Py_TRASHCAN_BEGIN_CONDITION((op), \ _PyTrash_cond(_PyObject_CAST(op), (destructor)(dealloc))) -/* The following two macros, Py_TRASHCAN_SAFE_BEGIN and - * Py_TRASHCAN_SAFE_END, are deprecated since version 3.11 and - * will be removed in the future. - * Use Py_TRASHCAN_BEGIN and Py_TRASHCAN_END instead. - */ -Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro; -#define Py_TRASHCAN_SAFE_BEGIN(op) \ - do { \ - UsingDeprecatedTrashcanMacro cond=1; \ - Py_TRASHCAN_BEGIN_CONDITION((op), cond); -#define Py_TRASHCAN_SAFE_END(op) \ - Py_TRASHCAN_END; \ - } while(0); PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); -PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); -PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj); +PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); +PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); #define TYPE_MAX_WATCHERS 8 diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 5a8cdd57c7845b..58a30aeea6ac64 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -78,14 +78,6 @@ PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator); PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj); -/* Code built with Py_BUILD_CORE must include pycore_gc.h instead which - defines a different _PyGC_FINALIZED() macro. */ -#ifndef Py_BUILD_CORE - // Kept for backward compatibility with Python 3.8 -# define _PyGC_FINALIZED(o) PyObject_GC_IsFinalized(o) -#endif - - // Test if a type supports weak references PyAPI_FUNC(int) PyType_SUPPORTS_WEAKREFS(PyTypeObject *type); diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h new file mode 100644 index 00000000000000..adc2c1fc442280 --- /dev/null +++ b/Include/cpython/optimizer.h @@ -0,0 +1,88 @@ + +#ifndef Py_LIMITED_API +#ifndef Py_OPTIMIZER_H +#define Py_OPTIMIZER_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _PyExecutorLinkListNode { + struct _PyExecutorObject *next; + struct _PyExecutorObject *previous; +} _PyExecutorLinkListNode; + + +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define BLOOM_FILTER_WORDS 8 + +typedef struct _bloom_filter { + uint32_t bits[BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + +typedef struct { + uint8_t opcode; + uint8_t oparg; + uint8_t valid; + uint8_t linked; + _PyBloomFilter bloom; + _PyExecutorLinkListNode links; +} _PyVMData; + +typedef struct _PyExecutorObject { + PyObject_VAR_HEAD + /* WARNING: execute consumes a reference to self. This is necessary to allow executors to tail call into each other. */ + struct _PyInterpreterFrame *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); + _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ + /* Data needed by the executor goes here, but is opaque to the VM */ +} _PyExecutorObject; + +typedef struct _PyOptimizerObject _PyOptimizerObject; + +/* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ +typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **, int curr_stackentries); + +typedef struct _PyOptimizerObject { + PyObject_HEAD + optimize_func optimize; + /* These thresholds are treated as signed so do not exceed INT16_MAX + * Use INT16_MAX to indicate that the optimizer should never be called */ + uint16_t resume_threshold; + uint16_t backedge_threshold; + /* Data needed by the optimizer goes here, but is opaque to the VM */ +} _PyOptimizerObject; + +PyAPI_FUNC(int) PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *executor); + +PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); + +PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); + +PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); + +int +_PyOptimizer_BackEdge(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest, PyObject **stack_pointer); + +extern _PyOptimizerObject _PyOptimizer_Default; + +void _Py_ExecutorInit(_PyExecutorObject *, _PyBloomFilter *); +void _Py_ExecutorClear(_PyExecutorObject *); +void _Py_BloomFilter_Init(_PyBloomFilter *); +void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); +PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); +extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp); + +/* For testing */ +PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); +PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); + +#define OPTIMIZER_BITS_IN_COUNTER 4 +/* Minimum of 16 additional executions before retry */ +#define MINIMUM_TIER2_BACKOFF 4 + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPTIMIZER_H */ +#endif /* Py_LIMITED_API */ diff --git a/Include/cpython/pthread_stubs.h b/Include/cpython/pthread_stubs.h index d95ee03d8308ce..e542eaa5bff0cf 100644 --- a/Include/cpython/pthread_stubs.h +++ b/Include/cpython/pthread_stubs.h @@ -21,13 +21,29 @@ #ifdef __wasi__ // WASI's bits/alltypes.h provides type definitions when __NEED_ is set. // The header file can be included multiple times. -# define __NEED_pthread_cond_t 1 -# define __NEED_pthread_condattr_t 1 -# define __NEED_pthread_mutex_t 1 -# define __NEED_pthread_mutexattr_t 1 -# define __NEED_pthread_key_t 1 -# define __NEED_pthread_t 1 -# define __NEED_pthread_attr_t 1 +// +// may also define these macros. +# ifndef __NEED_pthread_cond_t +# define __NEED_pthread_cond_t 1 +# endif +# ifndef __NEED_pthread_condattr_t +# define __NEED_pthread_condattr_t 1 +# endif +# ifndef __NEED_pthread_mutex_t +# define __NEED_pthread_mutex_t 1 +# endif +# ifndef __NEED_pthread_mutexattr_t +# define __NEED_pthread_mutexattr_t 1 +# endif +# ifndef __NEED_pthread_key_t +# define __NEED_pthread_key_t 1 +# endif +# ifndef __NEED_pthread_t +# define __NEED_pthread_t 1 +# endif +# ifndef __NEED_pthread_attr_t +# define __NEED_pthread_attr_t 1 +# endif # include #else typedef struct { void *__x; } pthread_cond_t; @@ -67,6 +83,7 @@ PyAPI_FUNC(int) pthread_create(pthread_t *restrict thread, void *(*start_routine)(void *), void *restrict arg); PyAPI_FUNC(int) pthread_detach(pthread_t thread); +PyAPI_FUNC(int) pthread_join(pthread_t thread, void** value_ptr); PyAPI_FUNC(pthread_t) pthread_self(void); PyAPI_FUNC(int) pthread_exit(void *retval) __attribute__ ((__noreturn__)); PyAPI_FUNC(int) pthread_attr_init(pthread_attr_t *attr); diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h new file mode 100644 index 00000000000000..5314a70436bfc3 --- /dev/null +++ b/Include/cpython/pyatomic.h @@ -0,0 +1,534 @@ +// This header provides cross-platform low-level atomic operations +// similar to C11 atomics. +// +// Operations are sequentially consistent unless they have a suffix indicating +// otherwise. If in doubt, prefer the sequentially consistent operations. +// +// The "_relaxed" suffix for load and store operations indicates the "relaxed" +// memory order. They don't provide synchronization, but (roughly speaking) +// guarantee somewhat sane behavior for races instead of undefined behavior. +// In practice, they correspond to "normal" hardware load and store +// instructions, so they are almost as inexpensive as plain loads and stores +// in C. +// +// Note that atomic read-modify-write operations like _Py_atomic_add_* return +// the previous value of the atomic variable, not the new value. +// +// See https://en.cppreference.com/w/c/atomic for more information on C11 +// atomics. +// See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf +// "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage +// or relaxed atomics. +// +// Functions with pseudo Python code: +// +// def _Py_atomic_load(obj): +// return obj # sequential consistency +// +// def _Py_atomic_load_relaxed(obj): +// return obj # relaxed consistency +// +// def _Py_atomic_store(obj, value): +// obj = value # sequential consistency +// +// def _Py_atomic_store_relaxed(obj, value): +// obj = value # relaxed consistency +// +// def _Py_atomic_exchange(obj, value): +// # sequential consistency +// old_obj = obj +// obj = value +// return old_obj +// +// def _Py_atomic_compare_exchange(obj, expected, desired): +// # sequential consistency +// if obj == expected: +// obj = desired +// return True +// else: +// expected = obj +// return False +// +// def _Py_atomic_add(obj, value): +// # sequential consistency +// old_obj = obj +// obj += value +// return old_obj +// +// def _Py_atomic_and(obj, value): +// # sequential consistency +// old_obj = obj +// obj &= value +// return old_obj +// +// def _Py_atomic_or(obj, value): +// # sequential consistency +// old_obj = obj +// obj |= value +// return old_obj +// +// Other functions: +// +// def _Py_atomic_load_ptr_acquire(obj): +// return obj # acquire +// +// def _Py_atomic_store_ptr_release(obj): +// return obj # release +// +// def _Py_atomic_fence_seq_cst(): +// # sequential consistency +// ... +// +// def _Py_atomic_fence_release(): +// # release +// ... + +#ifndef Py_CPYTHON_ATOMIC_H +# error "this header file must not be included directly" +#endif + +// --- _Py_atomic_add -------------------------------------------------------- +// Atomically adds `value` to `obj` and returns the previous value + +static inline int +_Py_atomic_add_int(int *obj, int value); + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value); + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value); + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value); + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value); + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value); + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value); + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value); + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value); + + +// --- _Py_atomic_compare_exchange ------------------------------------------- +// Performs an atomic compare-and-exchange. +// +// - If `*obj` and `*expected` are equal, store `desired` into `*obj` +// and return 1 (success). +// - Otherwise, store the `*obj` current value into `*expected` +// and return 0 (failure). +// +// These correspond to the C11 atomic_compare_exchange_strong() function. + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired); + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired); + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired); + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired); + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired); + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired); + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired); + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired); + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired); + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired); + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired); + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired); + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired); + +// NOTE: `obj` and `expected` are logically `void**` types, but we use `void*` +// so that we can pass types like `PyObject**` without a cast. +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value); + + +// --- _Py_atomic_exchange --------------------------------------------------- +// Atomically replaces `*obj` with `value` and returns the previous value of `*obj`. + +static inline int +_Py_atomic_exchange_int(int *obj, int value); + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value); + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value); + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value); + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value); + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value); + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value); + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value); + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value); + +static inline void * +_Py_atomic_exchange_ptr(void *obj, void *value); + + +// --- _Py_atomic_and -------------------------------------------------------- +// Performs `*obj &= value` atomically and returns the previous value of `*obj`. + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value); + + +// --- _Py_atomic_or --------------------------------------------------------- +// Performs `*obj |= value` atomically and returns the previous value of `*obj`. + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value); + + +// --- _Py_atomic_load ------------------------------------------------------- +// Atomically loads `*obj` (sequential consistency) + +static inline int +_Py_atomic_load_int(const int *obj); + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj); + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj); + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj); + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj); + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj); + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj); + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj); + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj); + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj); + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj); + +static inline void * +_Py_atomic_load_ptr(const void *obj); + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- +// Loads `*obj` (relaxed consistency, i.e., no ordering) + +static inline int +_Py_atomic_load_int_relaxed(const int *obj); + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj); + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj); + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj); + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj); + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj); + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj); + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj); + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj); + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj); + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj); + +static inline void * +_Py_atomic_load_ptr_relaxed(const void *obj); + + +// --- _Py_atomic_store ------------------------------------------------------ +// Atomically performs `*obj = value` (sequential consistency) + +static inline void +_Py_atomic_store_int(int *obj, int value); + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value); + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value); + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value); + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value); + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value); + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value); + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value); + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value); + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value); + +static inline void +_Py_atomic_store_ptr(void *obj, void *value); + +static inline void +_Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value); + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- +// Stores `*obj = value` (relaxed consistency, i.e., no ordering) + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value); + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value); + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value); + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value); + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value); + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value); + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value); + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value); + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value); + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value); + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value); + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +// Loads `*obj` (acquire operation) +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj); + +// Stores `*obj = value` (release operation) +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value); + +static inline void +_Py_atomic_store_int_release(int *obj, int value); + +static inline int +_Py_atomic_load_int_acquire(const int *obj); + + +// --- _Py_atomic_fence ------------------------------------------------------ + +// Sequential consistency fence. C11 fences have complex semantics. When +// possible, use the atomic operations on variables defined above, which +// generally do not require explicit use of a fence. +// See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence +static inline void _Py_atomic_fence_seq_cst(void); + +// Release fence +static inline void _Py_atomic_fence_release(void); + + +#ifndef _Py_USE_GCC_BUILTIN_ATOMICS +# if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) +# define _Py_USE_GCC_BUILTIN_ATOMICS 1 +# elif defined(__clang__) +# if __has_builtin(__atomic_load) +# define _Py_USE_GCC_BUILTIN_ATOMICS 1 +# endif +# endif +#endif + +#if _Py_USE_GCC_BUILTIN_ATOMICS +# define Py_ATOMIC_GCC_H +# include "cpython/pyatomic_gcc.h" +# undef Py_ATOMIC_GCC_H +#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) +# define Py_ATOMIC_STD_H +# include "cpython/pyatomic_std.h" +# undef Py_ATOMIC_STD_H +#elif defined(_MSC_VER) +# define Py_ATOMIC_MSC_H +# include "cpython/pyatomic_msc.h" +# undef Py_ATOMIC_MSC_H +#else +# error "no available pyatomic implementation for this platform/compiler" +#endif + + +// --- aliases --------------------------------------------------------------- + +#if SIZEOF_LONG == 8 +# define _Py_atomic_load_ulong(p) \ + _Py_atomic_load_uint64((uint64_t *)p) +# define _Py_atomic_load_ulong_relaxed(p) \ + _Py_atomic_load_uint64_relaxed((uint64_t *)p) +# define _Py_atomic_store_ulong(p, v) \ + _Py_atomic_store_uint64((uint64_t *)p, v) +# define _Py_atomic_store_ulong_relaxed(p, v) \ + _Py_atomic_store_uint64_relaxed((uint64_t *)p, v) +#elif SIZEOF_LONG == 4 +# define _Py_atomic_load_ulong(p) \ + _Py_atomic_load_uint32((uint32_t *)p) +# define _Py_atomic_load_ulong_relaxed(p) \ + _Py_atomic_load_uint32_relaxed((uint32_t *)p) +# define _Py_atomic_store_ulong(p, v) \ + _Py_atomic_store_uint32((uint32_t *)p, v) +# define _Py_atomic_store_ulong_relaxed(p, v) \ + _Py_atomic_store_uint32_relaxed((uint32_t *)p, v) +#else +# error "long must be 4 or 8 bytes in size" +#endif // SIZEOF_LONG diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h new file mode 100644 index 00000000000000..70f2b7e1b5706a --- /dev/null +++ b/Include/cpython/pyatomic_gcc.h @@ -0,0 +1,507 @@ +// This is the implementation of Python atomic operations using GCC's built-in +// functions that match the C+11 memory model. This implementation is preferred +// for GCC compatible compilers, such as Clang. These functions are available +// in GCC 4.8+ without needing to compile with --std=c11 or --std=gnu11. + +#ifndef Py_ATOMIC_GCC_H +# error "this header file must not be included directly" +#endif + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) +{ return __atomic_compare_exchange_n((void **)obj, (void **)expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void * +_Py_atomic_exchange_ptr(void *obj, void *value) +{ return __atomic_exchange_n((void **)obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline int +_Py_atomic_load_int(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline void * +_Py_atomic_load_ptr(const void *obj) +{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline void * +_Py_atomic_load_ptr_relaxed(const void *obj) +{ return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); } + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); } + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + + +// --- _Py_atomic_fence ------------------------------------------------------ + +static inline void +_Py_atomic_fence_seq_cst(void) +{ __atomic_thread_fence(__ATOMIC_SEQ_CST); } + + static inline void +_Py_atomic_fence_release(void) +{ __atomic_thread_fence(__ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h new file mode 100644 index 00000000000000..601a0cf65afc1c --- /dev/null +++ b/Include/cpython/pyatomic_msc.h @@ -0,0 +1,970 @@ +// This is the implementation of Python atomic operations for MSVC if the +// compiler does not support C11 or C++11 atomics. +// +// MSVC intrinsics are defined on char, short, long, __int64, and pointer +// types. Note that long and int are both 32-bits even on 64-bit Windows, +// so operations on int are cast to long. +// +// The volatile keyword has additional memory ordering semantics on MSVC. On +// x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64, +// volatile accesses behave like C11's memory_order_relaxed. + +#ifndef Py_ATOMIC_MSC_H +# error "this header file must not be included directly" +#endif + +#include + +#define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \ + Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE)) + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value); +} + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value); +} + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value); +} + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + int64_t new_value = old_value + value; + if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value); +} + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value); +} + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); +} + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(int64_t); + return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +#endif +} + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); +} + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + int8_t initial = (int8_t)_InterlockedCompareExchange8( + (volatile char *)obj, + (char)value, + (char)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + int16_t initial = (int16_t)_InterlockedCompareExchange16( + (volatile short *)obj, + (short)value, + (short)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + int32_t initial = (int32_t)_InterlockedCompareExchange( + (volatile long *)obj, + (long)value, + (long)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(__int64); + int64_t initial = (int64_t)_InterlockedCompareExchange64( + (volatile __int64 *)obj, + (__int64)value, + (__int64)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value) +{ + void *initial = _InterlockedCompareExchangePointer( + (void**)obj, + value, + *(void**)expected); + if (initial == *(void**)expected) { + return 1; + } + *(void**)expected = initial; + return 0; +} + + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value) +{ + return _Py_atomic_compare_exchange_int8((int8_t *)obj, + (int8_t *)expected, + (int8_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value) +{ + return _Py_atomic_compare_exchange_int16((int16_t *)obj, + (int16_t *)expected, + (int16_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value) +{ + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value) +{ + return _Py_atomic_compare_exchange_int64((int64_t *)obj, + (int64_t *)expected, + (int64_t)value); +} + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value); +} + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value); +} + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value); +} + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) { + return old_value; + } + } +#endif +} + +static inline void* +_Py_atomic_exchange_ptr(void *obj, void *value) +{ + return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value); +} + + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj, + (int8_t)value); +} + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj, + (int16_t)value); +} + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj, + (int64_t)value); +} + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (intptr_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value); +} + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value); +} + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value); +} + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); + for (;;) { + uint64_t new_value = old_value & value; + if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(uint64_t); + return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj, + (uint64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value); +} + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value); +} + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value); +} + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); + for (;;) { + uint64_t new_value = old_value | value; + if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(uint64_t); + return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj, + (uint64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint8_t *)obj; +#elif defined(_M_ARM64) + return (uint8_t)__ldar8((unsigned __int8 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint8" +#endif +} + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint16_t *)obj; +#elif defined(_M_ARM64) + return (uint16_t)__ldar16((unsigned __int16 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint16" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint32_t *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint64_t *)obj; +#elif defined(_M_ARM64) + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64" +#endif +} + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ + return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj); +} + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ + return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj); +} + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ + return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj); +} + +static inline int +_Py_atomic_load_int(const int *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (int)_Py_atomic_load_uint32((uint32_t *)obj); +} + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj); +} + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ + return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj); +} + +static inline void* +_Py_atomic_load_ptr(const void *obj) +{ +#if SIZEOF_VOID_P == 8 + return (void*)_Py_atomic_load_uint64((const uint64_t *)obj); +#else + return (void*)_Py_atomic_load_uint32((const uint32_t *)obj); +#endif +} + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (intptr_t)_Py_atomic_load_ptr((void*)obj); +} + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (uintptr_t)_Py_atomic_load_ptr((void*)obj); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj); +} + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ + return *(volatile int *)obj; +} + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ + return *(volatile int8_t *)obj; +} + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ + return *(volatile int16_t *)obj; +} + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ + return *(volatile int32_t *)obj; +} + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ + return *(volatile int64_t *)obj; +} + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ + return *(volatile intptr_t *)obj; +} + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ + return *(volatile uint8_t *)obj; +} + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ + return *(volatile uint16_t *)obj; +} + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ + return *(volatile uint32_t *)obj; +} + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ + return *(volatile uint64_t *)obj; +} + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ + return *(volatile uintptr_t *)obj; +} + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ + return *(volatile unsigned int *)obj; +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ + return *(volatile Py_ssize_t *)obj; +} + +static inline void* +_Py_atomic_load_ptr_relaxed(const void *obj) +{ + return *(void * volatile *)obj; +} + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ + (void)_Py_atomic_exchange_int(obj, value); +} + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ + (void)_Py_atomic_exchange_int8(obj, value); +} + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ + (void)_Py_atomic_exchange_int16(obj, value); +} + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ + (void)_Py_atomic_exchange_int32(obj, value); +} + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ + (void)_Py_atomic_exchange_int64(obj, value); +} + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ + (void)_Py_atomic_exchange_intptr(obj, value); +} + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ + (void)_Py_atomic_exchange_uint8(obj, value); +} + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ + (void)_Py_atomic_exchange_uint16(obj, value); +} + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ + (void)_Py_atomic_exchange_uint32(obj, value); +} + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ + (void)_Py_atomic_exchange_uint64(obj, value); +} + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ + (void)_Py_atomic_exchange_uintptr(obj, value); +} + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ + (void)_Py_atomic_exchange_uint(obj, value); +} + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ + (void)_Py_atomic_exchange_ptr(obj, value); +} + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + (void)_Py_atomic_exchange_ssize(obj, value); +} + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ + *(volatile int *)obj = value; +} + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ + *(volatile int8_t *)obj = value; +} + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ + *(volatile int16_t *)obj = value; +} + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ + *(volatile int32_t *)obj = value; +} + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ + *(volatile int64_t *)obj = value; +} + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + *(volatile intptr_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + *(volatile uint8_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + *(volatile uint16_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + *(volatile uint32_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + *(volatile uint64_t *)obj = value; +} + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + *(volatile uintptr_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void* value) +{ + *(void * volatile *)obj = value; +} + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + *(volatile Py_ssize_t *)obj = value; +} + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(void * volatile *)obj; +#elif defined(_M_ARM64) + return (void *)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ptr_acquire" +#endif +} + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(void * volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); +#else +# error "no implementation of _Py_atomic_store_ptr_release" +#endif +} + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_int_release" +#endif +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(int volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + return (int)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_int_acquire" +#endif +} + + +// --- _Py_atomic_fence ------------------------------------------------------ + + static inline void +_Py_atomic_fence_seq_cst(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) + __faststorefence(); +#elif defined(_M_IX86) + _mm_mfence(); +#else +# error "no implementation of _Py_atomic_fence_seq_cst" +#endif +} + + static inline void +_Py_atomic_fence_release(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadWriteBarrier(); +#else +# error "no implementation of _Py_atomic_fence_release" +#endif +} + +#undef _Py_atomic_ASSERT_ARG_TYPE diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h new file mode 100644 index 00000000000000..a05bfaec47e89d --- /dev/null +++ b/Include/cpython/pyatomic_std.h @@ -0,0 +1,889 @@ +// This is the implementation of Python atomic operations using C++11 or C11 +// atomics. Note that the pyatomic_gcc.h implementation is preferred for GCC +// compatible compilers, even if they support C++11 atomics. + +#ifndef Py_ATOMIC_STD_H +# error "this header file must not be included directly" +#endif + +#ifdef __cplusplus +extern "C++" { +# include +} +# define _Py_USING_STD using namespace std +# define _Atomic(tp) atomic +#else +# define _Py_USING_STD +# include +#endif + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int)*)obj, value); +} + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int8_t)*)obj, value); +} + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int16_t)*)obj, value); +} + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int32_t)*)obj, value); +} + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int64_t)*)obj, value); +} + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(intptr_t)*)obj, value); +} + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(unsigned int)*)obj, value); +} + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uintptr_t)*)obj, value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(Py_ssize_t)*)obj, value); +} + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int16_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int32_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int64_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(intptr_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(unsigned int)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint8_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint16_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint32_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint64_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uintptr_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(Py_ssize_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(void *)*)obj, + (void **)expected, desired); +} + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int)*)obj, value); +} + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int8_t)*)obj, value); +} + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int16_t)*)obj, value); +} + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int32_t)*)obj, value); +} + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int64_t)*)obj, value); +} + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(intptr_t)*)obj, value); +} + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(unsigned int)*)obj, value); +} + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uintptr_t)*)obj, value); +} + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(Py_ssize_t)*)obj, value); +} + +static inline void* +_Py_atomic_exchange_ptr(void *obj, void *value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(void *)*)obj, value); +} + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uintptr_t)*)obj, value); +} + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uintptr_t)*)obj, value); +} + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline int +_Py_atomic_load_int(const int *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int)*)obj); +} + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int8_t)*)obj); +} + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int16_t)*)obj); +} + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int32_t)*)obj); +} + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int64_t)*)obj); +} + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(intptr_t)*)obj); +} + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint8_t)*)obj); +} + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint32_t)*)obj); +} + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint32_t)*)obj); +} + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint64_t)*)obj); +} + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uintptr_t)*)obj); +} + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(unsigned int)*)obj); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(Py_ssize_t)*)obj); +} + +static inline void* +_Py_atomic_load_ptr(const void *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(void*)*)obj); +} + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int)*)obj, + memory_order_relaxed); +} + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int8_t)*)obj, + memory_order_relaxed); +} + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int16_t)*)obj, + memory_order_relaxed); +} + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int32_t)*)obj, + memory_order_relaxed); +} + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int64_t)*)obj, + memory_order_relaxed); +} + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(intptr_t)*)obj, + memory_order_relaxed); +} + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint8_t)*)obj, + memory_order_relaxed); +} + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint16_t)*)obj, + memory_order_relaxed); +} + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_relaxed); +} + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_relaxed); +} + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_relaxed); +} + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned int)*)obj, + memory_order_relaxed); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_relaxed); +} + +static inline void* +_Py_atomic_load_ptr_relaxed(const void *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(void*)*)obj, + memory_order_relaxed); +} + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int)*)obj, value); +} + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int8_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int16_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int32_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int64_t)*)obj, value); +} + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(intptr_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint8_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint16_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint32_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint64_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uintptr_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store((_Atomic(unsigned int)*)obj, value); +} + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store((_Atomic(void*)*)obj, value); +} + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(Py_ssize_t)*)obj, value); +} + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int8_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int16_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int32_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int64_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(intptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint8_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint16_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(void*)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_relaxed); +} + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(void*)*)obj, + memory_order_acquire); +} + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(void*)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int)*)obj, value, + memory_order_release); +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int)*)obj, + memory_order_acquire); +} + + + +// --- _Py_atomic_fence ------------------------------------------------------ + + static inline void +_Py_atomic_fence_seq_cst(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_seq_cst); +} + + static inline void +_Py_atomic_fence_release(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_release); +} diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 758804ade2baa7..479b908fb7058a 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -88,38 +88,15 @@ typedef PyOSErrorObject PyEnvironmentErrorObject; typedef PyOSErrorObject PyWindowsErrorObject; #endif -/* Error handling definitions */ - -PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *); -PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate); -PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *); -PyAPI_FUNC(void) _PyErr_SetHandledException(PyThreadState *, PyObject *); -PyAPI_FUNC(void) _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); - -/* Context manipulation (PEP 3134) */ - -Py_DEPRECATED(3.12) PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); -PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); - -/* Like PyErr_Format(), but saves current exception as __context__ and - __cause__. - */ -PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause( - PyObject *exception, - const char *format, /* ASCII-encoded string */ - ... - ); - /* In exceptions.c */ -PyAPI_FUNC(int) _PyException_AddNote( - PyObject *exc, - PyObject *note); +PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( + PyObject *orig, + PyObject *excs); /* In signalmodule.c */ int PySignal_SetWakeupFd(int fd); -PyAPI_FUNC(int) _PyErr_CheckSignals(void); /* Support for adding program text to SyntaxErrors */ @@ -139,36 +116,10 @@ PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject( PyObject *filename, int lineno); -PyAPI_FUNC(PyObject *) _PyErr_ProgramDecodedTextObject( - PyObject *filename, - int lineno, - const char* encoding); - -PyAPI_FUNC(PyObject *) _PyUnicodeTranslateError_Create( - PyObject *object, - Py_ssize_t start, - Py_ssize_t end, - const char *reason /* UTF-8 encoded string */ - ); - -PyAPI_FUNC(void) _PyErr_WriteUnraisableMsg( - const char *err_msg, - PyObject *obj); - PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFunc( const char *func, const char *message); -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFormat( - const char *func, - const char *format, - ...); - -extern PyObject *_PyErr_SetImportErrorWithNameFrom( - PyObject *, - PyObject *, - PyObject *, - PyObject *); - +PyAPI_FUNC(void) PyErr_FormatUnraisable(const char *, ...); #define Py_FatalError(message) _Py_FatalErrorFunc(__func__, (message)) diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index 6ec292718aff1a..c5adbbe4868f69 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -16,3 +16,28 @@ PyAPI_FUNC(PyObject *) PyFrame_GetGenerator(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_GetLasti(PyFrameObject *frame); PyAPI_FUNC(PyObject*) PyFrame_GetVar(PyFrameObject *frame, PyObject *name); PyAPI_FUNC(PyObject*) PyFrame_GetVarString(PyFrameObject *frame, const char *name); + +/* The following functions are for use by debuggers and other tools + * implementing custom frame evaluators with PEP 523. */ + +struct _PyInterpreterFrame; + +/* Returns the code object of the frame (strong reference). + * Does not raise an exception. */ +PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); + +/* Returns a byte ofsset into the last executed instruction. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); + +/* Returns the currently executing line number, or -1 if there is no line number. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); + +#define PyUnstable_EXECUTABLE_KIND_SKIP 0 +#define PyUnstable_EXECUTABLE_KIND_PY_FUNCTION 1 +#define PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION 3 +#define PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4 +#define PyUnstable_EXECUTABLE_KINDS 5 + +PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1]; diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h new file mode 100644 index 00000000000000..396c208e1b106a --- /dev/null +++ b/Include/cpython/pyhash.h @@ -0,0 +1,39 @@ +#ifndef Py_CPYTHON_HASH_H +# error "this header file must not be included directly" +#endif + +/* Prime multiplier used in string and various other hashes. */ +#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ + +/* Parameters used for the numeric hash implementation. See notes for + _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on + reduction modulo the prime 2**_PyHASH_BITS - 1. */ + +#if SIZEOF_VOID_P >= 8 +# define _PyHASH_BITS 61 +#else +# define _PyHASH_BITS 31 +#endif + +#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define _PyHASH_INF 314159 +#define _PyHASH_IMAG _PyHASH_MULTIPLIER + +/* Helpers for hash functions */ +PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); + +// Kept for backward compatibility +#define _Py_HashPointer Py_HashPointer + + +/* hash function definition */ +typedef struct { + Py_hash_t (*const hash)(const void *, Py_ssize_t); + const char *name; + const int hash_bits; + const int seed_bits; +} PyHash_FuncDef; + +PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); + +PyAPI_FUNC(Py_hash_t) Py_HashPointer(const void *ptr); diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 08569ee683ce0d..d425a233f71000 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -6,13 +6,6 @@ in all builds of Python */ PyAPI_FUNC(int) Py_FrozenMain(int argc, char **argv); -/* Only used by applications that embed the interpreter and need to - * override the standard encoding determination mechanism - */ -Py_DEPRECATED(3.11) PyAPI_FUNC(int) Py_SetStandardStreamEncoding( - const char *encoding, - const char *errors); - /* PEP 432 Multi-phase initialization API (Private while provisional!) */ PyAPI_FUNC(PyStatus) Py_PreInitialize( @@ -26,13 +19,13 @@ PyAPI_FUNC(PyStatus) Py_PreInitializeFromArgs( Py_ssize_t argc, wchar_t **argv); -PyAPI_FUNC(int) _Py_IsCoreInitialized(void); - /* Initialization and finalization */ PyAPI_FUNC(PyStatus) Py_InitializeFromConfig( const PyConfig *config); + +// Python 3.8 provisional API (PEP 587) PyAPI_FUNC(PyStatus) _Py_InitializeMain(void); PyAPI_FUNC(int) Py_RunMain(void); @@ -40,32 +33,51 @@ PyAPI_FUNC(int) Py_RunMain(void); PyAPI_FUNC(void) _Py_NO_RETURN Py_ExitStatusException(PyStatus err); -/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ -PyAPI_FUNC(void) _Py_RestoreSignals(void); - PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); -PyAPI_FUNC(int) _Py_FdIsInteractive(FILE *fp, PyObject *filename); - -Py_DEPRECATED(3.11) PyAPI_FUNC(void) _Py_SetProgramFullPath(const wchar_t *); - -PyAPI_FUNC(const char *) _Py_gitidentifier(void); -PyAPI_FUNC(const char *) _Py_gitversion(void); - -PyAPI_FUNC(int) _Py_IsFinalizing(void); - -/* Random */ -PyAPI_FUNC(int) _PyOS_URandom(void *buffer, Py_ssize_t size); -PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size); -/* Legacy locale support */ -PyAPI_FUNC(int) _Py_CoerceLegacyLocale(int warn); -PyAPI_FUNC(int) _Py_LegacyLocaleDetected(int warn); -PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); +/* --- PyInterpreterConfig ------------------------------------ */ + +#define PyInterpreterConfig_DEFAULT_GIL (0) +#define PyInterpreterConfig_SHARED_GIL (1) +#define PyInterpreterConfig_OWN_GIL (2) + +typedef struct { + // XXX "allow_object_sharing"? "own_objects"? + int use_main_obmalloc; + int allow_fork; + int allow_exec; + int allow_threads; + int allow_daemon_threads; + int check_multi_interp_extensions; + int gil; +} PyInterpreterConfig; + +#define _PyInterpreterConfig_INIT \ + { \ + .use_main_obmalloc = 0, \ + .allow_fork = 0, \ + .allow_exec = 0, \ + .allow_threads = 1, \ + .allow_daemon_threads = 0, \ + .check_multi_interp_extensions = 1, \ + .gil = PyInterpreterConfig_OWN_GIL, \ + } + +#define _PyInterpreterConfig_LEGACY_INIT \ + { \ + .use_main_obmalloc = 1, \ + .allow_fork = 1, \ + .allow_exec = 1, \ + .allow_threads = 1, \ + .allow_daemon_threads = 1, \ + .check_multi_interp_extensions = 0, \ + .gil = PyInterpreterConfig_SHARED_GIL, \ + } PyAPI_FUNC(PyStatus) Py_NewInterpreterFromConfig( PyThreadState **tstate_p, const PyInterpreterConfig *config); typedef void (*atexit_datacallbackfunc)(void *); -PyAPI_FUNC(int) _Py_AtExit( +PyAPI_FUNC(int) PyUnstable_AtExit( PyInterpreterState *, atexit_datacallbackfunc, void *); diff --git a/Include/cpython/pymem.h b/Include/cpython/pymem.h index d1054d76520b9a..76b3221f7b9f39 100644 --- a/Include/cpython/pymem.h +++ b/Include/cpython/pymem.h @@ -2,24 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); -PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); -PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); -PyAPI_FUNC(void) PyMem_RawFree(void *ptr); - -/* Try to get the allocators name set by _PyMem_SetupAllocators(). */ -PyAPI_FUNC(const char*) _PyMem_GetCurrentAllocatorName(void); - -/* strdup() using PyMem_RawMalloc() */ -PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); - -/* strdup() using PyMem_Malloc() */ -PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); - -/* wcsdup() using PyMem_RawMalloc() */ -PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str); - - typedef enum { /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */ PYMEM_DOMAIN_RAW, @@ -41,6 +23,10 @@ typedef enum { PYMEM_ALLOCATOR_PYMALLOC = 5, PYMEM_ALLOCATOR_PYMALLOC_DEBUG = 6, #endif +#ifdef WITH_MIMALLOC + PYMEM_ALLOCATOR_MIMALLOC = 7, + PYMEM_ALLOCATOR_MIMALLOC_DEBUG = 8, +#endif } PyMemAllocatorName; diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index f33c72d4cf4d2a..56172d231c44f4 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -3,45 +3,12 @@ #endif -/* -Runtime Feature Flags - -Each flag indicate whether or not a specific runtime feature -is available in a given context. For example, forking the process -might not be allowed in the current interpreter (i.e. os.fork() would fail). -*/ - -/* Set if the interpreter share obmalloc runtime state - with the main interpreter. */ -#define Py_RTFLAGS_USE_MAIN_OBMALLOC (1UL << 5) - -/* Set if import should check a module for subinterpreter support. */ -#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8) - -/* Set if threads are allowed. */ -#define Py_RTFLAGS_THREADS (1UL << 10) - -/* Set if daemon threads are allowed. */ -#define Py_RTFLAGS_DAEMON_THREADS (1UL << 11) - -/* Set if os.fork() is allowed. */ -#define Py_RTFLAGS_FORK (1UL << 15) - -/* Set if os.exec*() is allowed. */ -#define Py_RTFLAGS_EXEC (1UL << 16) - - -PyAPI_FUNC(int) _PyInterpreterState_HasFeature(PyInterpreterState *interp, - unsigned long feature); - - /* private interpreter helpers */ PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int); -PyAPI_FUNC(PyObject *) _PyInterpreterState_GetMainModule(PyInterpreterState *); - +PyAPI_FUNC(PyObject *) PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *); /* State unique per thread */ @@ -62,24 +29,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 -// Internal structure: you should not use it directly, but use public functions -// like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing(). -typedef struct _PyCFrame { - /* This struct will be threaded through the C stack - * allowing fast access to per-thread state that needs - * to be accessed quickly by the interpreter, but can - * be modified outside of the interpreter. - * - * WARNING: This makes data on the C stack accessible from - * heap objects. Care must be taken to maintain stack - * discipline and make sure that instances of this struct cannot - * accessed outside of their lifetime. - */ - /* Pointer to the currently executing frame (it can be NULL) */ - struct _PyInterpreterFrame *current_frame; - struct _PyCFrame *previous; -} _PyCFrame; - typedef struct _err_stackitem { /* This struct represents a single execution context where we might * be currently handling an exception. It is a per-coroutine state @@ -143,6 +92,19 @@ struct _ts { /* padding to align to 4 bytes */ unsigned int :24; } _status; +#ifdef Py_BUILD_CORE +# define _PyThreadState_WHENCE_NOTSET -1 +# define _PyThreadState_WHENCE_UNKNOWN 0 +# define _PyThreadState_WHENCE_INTERP 1 +# define _PyThreadState_WHENCE_THREADING 2 +# define _PyThreadState_WHENCE_GILSTATE 3 +# define _PyThreadState_WHENCE_EXEC 4 +#endif + int _whence; + + /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_GC). + See Include/internal/pycore_pystate.h for more details. */ + int state; int py_recursion_remaining; int py_recursion_limit; @@ -156,9 +118,8 @@ struct _ts { int tracing; int what_event; /* The event currently being monitored, if any. */ - /* Pointer to current _PyCFrame in the C stack frame of the currently, - * or most recently, executing _PyEval_EvalFrameDefault. */ - _PyCFrame *cframe; + /* Pointer to currently executing frame. */ + struct _PyInterpreterFrame *current_frame; Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; @@ -188,6 +149,13 @@ struct _ts { struct _py_trashcan trash; + /* Tagged pointer to top-most critical section, or zero if there is no + * active critical section. Critical sections are only used in + * `--disable-gil` builds (i.e., when Py_GIL_DISABLED is defined to 1). In the + * default build, this field is always zero. + */ + uintptr_t critical_section; + /* Called when a thread state is deleted normally, but not when it * is destroyed after fork(). * Pain: to prevent rare but fatal shutdown errors (issue 18808), @@ -244,35 +212,32 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; - /* The bottom-most frame on the stack. */ - _PyCFrame root_cframe; }; -/* WASI has limited call stack. Python's recursion limit depends on code - layout, optimization, and WASI runtime. Wasmtime can handle about 700 - recursions, sometimes less. 500 is a more conservative limit. */ -#ifndef C_RECURSION_LIMIT -# ifdef __wasi__ -# define C_RECURSION_LIMIT 500 -# else -# define C_RECURSION_LIMIT 800 -# endif +#ifdef Py_DEBUG + // A debug build is likely built with low optimization level which implies + // higher stack memory usage than a release build: use a lower limit. +# define Py_C_RECURSION_LIMIT 500 +#elif defined(__wasi__) + // WASI has limited call stack. Python's recursion limit depends on code + // layout, optimization, and WASI runtime. Wasmtime can handle about 700 + // recursions, sometimes less. 500 is a more conservative limit. +# define Py_C_RECURSION_LIMIT 500 +#else + // This value is duplicated in Lib/test/support/__init__.py +# define Py_C_RECURSION_LIMIT 1500 #endif -/* other API */ -// Alias for backward compatibility with Python 3.8 -#define _PyInterpreterState_Get PyInterpreterState_Get - -/* An alias for the internal _PyThreadState_New(), - kept for stable ABI compatibility. */ -PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *); +/* other API */ /* Similar to PyThreadState_Get(), but don't issue a fatal error * if it is NULL. */ -PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); +PyAPI_FUNC(PyThreadState *) PyThreadState_GetUnchecked(void); + +// Alias kept for backward compatibility +#define _PyThreadState_UncheckedGet PyThreadState_GetUnchecked -PyAPI_FUNC(PyObject *) _PyThreadState_GetDict(PyThreadState *tstate); // Disable tracing and profiling. PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); @@ -289,24 +254,10 @@ PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); The function returns 1 if _PyGILState_check_enabled is non-zero. */ PyAPI_FUNC(int) PyGILState_Check(void); -/* Get the single PyInterpreterState used by this process' GILState - implementation. - - This function doesn't check for error. Return NULL before _PyGILState_Init() - is called and after _PyGILState_Fini() is called. - - See also _PyInterpreterState_Get() and _PyInterpreterState_GET(). */ -PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); - /* The implementation of sys._current_frames() Returns a dict mapping thread id to that thread's current frame. */ -PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void); - -/* The implementation of sys._current_exceptions() Returns a dict mapping - thread id to that thread's current exception. -*/ -PyAPI_FUNC(PyObject *) _PyThread_CurrentExceptions(void); +PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ @@ -326,118 +277,3 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); - -PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp); - -/* Get a copy of the current interpreter configuration. - - Return 0 on success. Raise an exception and return -1 on error. - - The caller must initialize 'config', using PyConfig_InitPythonConfig() - for example. - - Python must be preinitialized to call this method. - The caller must hold the GIL. - - Once done with the configuration, PyConfig_Clear() must be called to clear - it. */ -PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( - struct PyConfig *config); - -/* Set the configuration of the current interpreter. - - This function should be called during or just after the Python - initialization. - - Update the sys module with the new configuration. If the sys module was - modified directly after the Python initialization, these changes are lost. - - Some configuration like faulthandler or warnoptions can be updated in the - configuration, but don't reconfigure Python (don't enable/disable - faulthandler and don't reconfigure warnings filters). - - Return 0 on success. Raise an exception and return -1 on error. - - The configuration should come from _PyInterpreterState_GetConfigCopy(). */ -PyAPI_FUNC(int) _PyInterpreterState_SetConfig( - const struct PyConfig *config); - -// Get the configuration of the current interpreter. -// The caller must hold the GIL. -PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); - - -/* cross-interpreter data */ - -// _PyCrossInterpreterData is similar to Py_buffer as an effectively -// opaque struct that holds data outside the object machinery. This -// is necessary to pass safely between interpreters in the same process. -typedef struct _xid _PyCrossInterpreterData; - -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); -typedef void (*xid_freefunc)(void *); - -struct _xid { - // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the - // new_object func (below) encodes the data. - void *data; - // obj is the Python object from which the data was derived. This - // is non-NULL only if the data remains bound to the object in some - // way, such that the object must be "released" (via a decref) when - // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for - // ensuring it owns the reference (i.e. incref). - PyObject *obj; - // interp is the ID of the owning interpreter of the original - // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only - // be set by the cross-interpreter machinery. - // - // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. Note that IDs are never re-used, so - // each one will always correspond to a specific interpreter - // (whether still alive or not). - int64_t interp; - // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object (a new - // reference) will be equivalent to the original object. This field - // is required. - xid_newobjectfunc new_object; - // free is called when the data is released. If it is NULL then - // nothing will be done to free the data. For some types this is - // okay (e.g. bytes) and for those types this field should be set - // to NULL. However, for most the data was allocated just for - // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will - // leak. In that case, at the very least this field should be set - // to PyMem_RawFree (the default if not explicitly set to NULL). - // The call will happen with the original interpreter activated. - xid_freefunc free; -}; - -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, - PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, - PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); - -/* cross-interpreter data registry */ - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h new file mode 100644 index 00000000000000..294bf1505f0115 --- /dev/null +++ b/Include/cpython/pystats.h @@ -0,0 +1,142 @@ +// Statistics on Python performance. +// +// API: +// +// - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() +// and Py_DECREF(). +// - _Py_stats variable +// +// Functions of the sys module: +// +// - sys._stats_on() +// - sys._stats_off() +// - sys._stats_clear() +// - sys._stats_dump() +// +// Python must be built with ./configure --enable-pystats to define the +// Py_STATS macro. +// +// Define _PY_INTERPRETER macro to increment interpreter_increfs and +// interpreter_decrefs. Otherwise, increment increfs and decrefs. + +#ifndef Py_CPYTHON_PYSTATS_H +# error "this header file must not be included directly" +#endif + +#define SPECIALIZATION_FAILURE_KINDS 36 + +/* Stats for determining who is calling PyEval_EvalFrame */ +#define EVAL_CALL_TOTAL 0 +#define EVAL_CALL_VECTOR 1 +#define EVAL_CALL_GENERATOR 2 +#define EVAL_CALL_LEGACY 3 +#define EVAL_CALL_FUNCTION_VECTORCALL 4 +#define EVAL_CALL_BUILD_CLASS 5 +#define EVAL_CALL_SLOT 6 +#define EVAL_CALL_FUNCTION_EX 7 +#define EVAL_CALL_API 8 +#define EVAL_CALL_METHOD 9 + +#define EVAL_CALL_KINDS 10 + +typedef struct _specialization_stats { + uint64_t success; + uint64_t failure; + uint64_t hit; + uint64_t deferred; + uint64_t miss; + uint64_t deopt; + uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS]; +} SpecializationStats; + +typedef struct _opcode_stats { + SpecializationStats specialization; + uint64_t execution_count; + uint64_t pair_count[256]; +} OpcodeStats; + +typedef struct _call_stats { + uint64_t inlined_py_calls; + uint64_t pyeval_calls; + uint64_t frames_pushed; + uint64_t frame_objects_created; + uint64_t eval_calls[EVAL_CALL_KINDS]; +} CallStats; + +typedef struct _object_stats { + uint64_t increfs; + uint64_t decrefs; + uint64_t interpreter_increfs; + uint64_t interpreter_decrefs; + uint64_t allocations; + uint64_t allocations512; + uint64_t allocations4k; + uint64_t allocations_big; + uint64_t frees; + uint64_t to_freelist; + uint64_t from_freelist; + uint64_t new_values; + uint64_t dict_materialized_on_request; + uint64_t dict_materialized_new_key; + uint64_t dict_materialized_too_big; + uint64_t dict_materialized_str_subclass; + uint64_t dict_dematerialized; + uint64_t type_cache_hits; + uint64_t type_cache_misses; + uint64_t type_cache_dunder_hits; + uint64_t type_cache_dunder_misses; + uint64_t type_cache_collisions; + /* Temporary value used during GC */ + uint64_t object_visits; +} ObjectStats; + +typedef struct _gc_stats { + uint64_t collections; + uint64_t object_visits; + uint64_t objects_collected; +} GCStats; + +typedef struct _uop_stats { + uint64_t execution_count; + uint64_t miss; +} UOpStats; + +#define _Py_UOP_HIST_SIZE 32 + +typedef struct _optimization_stats { + uint64_t attempts; + uint64_t traces_created; + uint64_t traces_executed; + uint64_t uops_executed; + uint64_t trace_stack_overflow; + uint64_t trace_stack_underflow; + uint64_t trace_too_long; + uint64_t trace_too_short; + uint64_t inner_loop; + uint64_t recursive_call; + UOpStats opcode[512]; + uint64_t unsupported_opcode[256]; + uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; +} OptimizationStats; + +typedef struct _stats { + OpcodeStats opcode_stats[256]; + CallStats call_stats; + ObjectStats object_stats; + OptimizationStats optimization_stats; + GCStats *gc_stats; +} PyStats; + + +// Export for shared extensions like 'math' +PyAPI_DATA(PyStats*) _Py_stats; + +#ifdef _PY_INTERPRETER +# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) +# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) +#else +# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) +# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) +#endif diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h index fb617655374026..edc40952254029 100644 --- a/Include/cpython/pythonrun.h +++ b/Include/cpython/pythonrun.h @@ -3,21 +3,11 @@ #endif PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); -PyAPI_FUNC(int) _PyRun_SimpleFileObject( - FILE *fp, - PyObject *filename, - int closeit, - PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_AnyFileExFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ int closeit, PyCompilerFlags *flags); -PyAPI_FUNC(int) _PyRun_AnyFileObject( - FILE *fp, - PyObject *filename, - int closeit, - PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_SimpleFileExFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ @@ -35,10 +25,6 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ PyCompilerFlags *flags); -PyAPI_FUNC(int) _PyRun_InteractiveLoopObject( - FILE *fp, - PyObject *filename, - PyCompilerFlags *flags); PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, @@ -69,15 +55,6 @@ PyAPI_FUNC(PyObject *) Py_CompileStringObject( #define Py_CompileString(str, p, s) Py_CompileStringExFlags((str), (p), (s), NULL, -1) #define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags((str), (p), (s), (f), -1) - -PyAPI_FUNC(const char *) _Py_SourceAsString( - PyObject *cmd, - const char *funcname, - const char *what, - PyCompilerFlags *cf, - PyObject **cmd_copy); - - /* A function flavor is also exported by libpython. It is required when libpython is accessed directly rather than using header files which defines macros below. On Windows, for example, PyAPI_FUNC() uses dllexport to @@ -114,8 +91,6 @@ PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *fp, const char *p, int s, PyObject #define PyRun_FileFlags(fp, p, s, g, l, flags) \ PyRun_FileExFlags((fp), (p), (s), (g), (l), 0, (flags)) - /* Stuff with no proper home (yet) */ PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *); -PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *); diff --git a/Include/cpython/pythread.h b/Include/cpython/pythread.h index ce4ec8f65b15ea..03f710a9f7ef2e 100644 --- a/Include/cpython/pythread.h +++ b/Include/cpython/pythread.h @@ -2,14 +2,15 @@ # error "this header file must not be included directly" #endif -#define PYTHREAD_INVALID_THREAD_ID ((unsigned long)-1) +// PY_TIMEOUT_MAX is the highest usable value (in microseconds) of PY_TIMEOUT_T +// type, and depends on the system threading API. +// +// NOTE: this isn't the same value as `_thread.TIMEOUT_MAX`. The _thread module +// exposes a higher-level API, with timeouts expressed in seconds and +// floating-point numbers allowed. +PyAPI_DATA(const long long) PY_TIMEOUT_MAX; -#ifdef HAVE_FORK -/* Private function to reinitialize a lock at fork in the child process. - Reset the lock to the unlocked state. - Return 0 on success, return -1 on error. */ -PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock); -#endif /* HAVE_FORK */ +#define PYTHREAD_INVALID_THREAD_ID ((unsigned long)-1) #ifdef HAVE_PTHREAD_H /* Darwin needs pthread.h to know type name the pthread_key_t. */ diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h deleted file mode 100644 index 16d88d191e9e25..00000000000000 --- a/Include/cpython/pytime.h +++ /dev/null @@ -1,331 +0,0 @@ -// The _PyTime_t API is written to use timestamp and timeout values stored in -// various formats and to read clocks. -// -// The _PyTime_t type is an integer to support directly common arithmetic -// operations like t1 + t2. -// -// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type -// is signed to support negative timestamps. The supported range is around -// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the -// supported date range is around [1677-09-21; 2262-04-11]. -// -// Formats: -// -// * seconds -// * seconds as a floating pointer number (C double) -// * milliseconds (10^-3 seconds) -// * microseconds (10^-6 seconds) -// * 100 nanoseconds (10^-7 seconds) -// * nanoseconds (10^-9 seconds) -// * timeval structure, 1 microsecond resolution (10^-6 seconds) -// * timespec structure, 1 nanosecond resolution (10^-9 seconds) -// -// Integer overflows are detected and raise OverflowError. Conversion to a -// resolution worse than 1 nanosecond is rounded correctly with the requested -// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling -// (towards +inf), half even and up (away from zero). -// -// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so -// the caller doesn't have to handle errors and doesn't need to hold the GIL. -// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on -// overflow. -// -// Clocks: -// -// * System clock -// * Monotonic clock -// * Performance counter -// -// Operations like (t * k / q) with integers are implemented in a way to reduce -// the risk of integer overflow. Such operation is used to convert a clock -// value expressed in ticks with a frequency to _PyTime_t, like -// QueryPerformanceCounter() with QueryPerformanceFrequency(). - -#ifndef Py_LIMITED_API -#ifndef Py_PYTIME_H -#define Py_PYTIME_H - -/************************************************************************** -Symbols and macros to supply platform-independent interfaces to time related -functions and constants -**************************************************************************/ -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __clang__ -struct timeval; -#endif - -/* _PyTime_t: Python timestamp with subsecond precision. It can be used to - store a duration, and so indirectly a date (related to another date, like - UNIX epoch). */ -typedef int64_t _PyTime_t; -// _PyTime_MIN nanoseconds is around -292.3 years -#define _PyTime_MIN INT64_MIN -// _PyTime_MAX nanoseconds is around +292.3 years -#define _PyTime_MAX INT64_MAX -#define _SIZEOF_PYTIME_T 8 - -typedef enum { - /* Round towards minus infinity (-inf). - For example, used to read a clock. */ - _PyTime_ROUND_FLOOR=0, - /* Round towards infinity (+inf). - For example, used for timeout to wait "at least" N seconds. */ - _PyTime_ROUND_CEILING=1, - /* Round to nearest with ties going to nearest even integer. - For example, used to round from a Python float. */ - _PyTime_ROUND_HALF_EVEN=2, - /* Round away from zero - For example, used for timeout. _PyTime_ROUND_CEILING rounds - -1e-9 to 0 milliseconds which causes bpo-31786 issue. - _PyTime_ROUND_UP rounds -1e-9 to -1 millisecond which keeps - the timeout sign as expected. select.poll(timeout) must block - for negative values." */ - _PyTime_ROUND_UP=3, - /* _PyTime_ROUND_TIMEOUT (an alias for _PyTime_ROUND_UP) should be - used for timeouts. */ - _PyTime_ROUND_TIMEOUT = _PyTime_ROUND_UP -} _PyTime_round_t; - - -/* Convert a time_t to a PyLong. */ -PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( - time_t sec); - -/* Convert a PyLong to a time_t. */ -PyAPI_FUNC(time_t) _PyLong_AsTime_t( - PyObject *obj); - -/* Convert a number of seconds, int or float, to time_t. */ -PyAPI_FUNC(int) _PyTime_ObjectToTime_t( - PyObject *obj, - time_t *sec, - _PyTime_round_t); - -/* Convert a number of seconds, int or float, to a timeval structure. - usec is in the range [0; 999999] and rounded towards zero. - For example, -1.2 is converted to (-2, 800000). */ -PyAPI_FUNC(int) _PyTime_ObjectToTimeval( - PyObject *obj, - time_t *sec, - long *usec, - _PyTime_round_t); - -/* Convert a number of seconds, int or float, to a timespec structure. - nsec is in the range [0; 999999999] and rounded towards zero. - For example, -1.2 is converted to (-2, 800000000). */ -PyAPI_FUNC(int) _PyTime_ObjectToTimespec( - PyObject *obj, - time_t *sec, - long *nsec, - _PyTime_round_t); - - -/* Create a timestamp from a number of seconds. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); - -/* Macro to create a timestamp from a number of seconds, no integer overflow. - Only use the macro for small values, prefer _PyTime_FromSeconds(). */ -#define _PYTIME_FROMSECONDS(seconds) \ - ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) - -/* Create a timestamp from a number of nanoseconds. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); - -/* Create a timestamp from a number of microseconds. - * Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromMicrosecondsClamp(_PyTime_t us); - -/* Create a timestamp from nanoseconds (Python int). */ -PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, - PyObject *obj); - -/* Convert a number of seconds (Python float or int) to a timestamp. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, - PyObject *obj, - _PyTime_round_t round); - -/* Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, - PyObject *obj, - _PyTime_round_t round); - -/* Convert a timestamp to a number of seconds as a C double. */ -PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); - -/* Convert timestamp to a number of milliseconds (10^-3 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, - _PyTime_round_t round); - -/* Convert timestamp to a number of microseconds (10^-6 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, - _PyTime_round_t round); - -/* Convert timestamp to a number of nanoseconds (10^-9 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsNanoseconds(_PyTime_t t); - -#ifdef MS_WINDOWS -// Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). -PyAPI_FUNC(_PyTime_t) _PyTime_As100Nanoseconds(_PyTime_t t, - _PyTime_round_t round); -#endif - -/* Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int - object. */ -PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t); - -#ifndef MS_WINDOWS -/* Create a timestamp from a timeval structure. - Raise an exception and return -1 on overflow, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); -#endif - -/* Convert a timestamp to a timeval structure (microsecond resolution). - tv_usec is always positive. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, - struct timeval *tv, - _PyTime_round_t round); - -/* Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. - On overflow, clamp tv_sec to _PyTime_t min/max. */ -PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, - struct timeval *tv, - _PyTime_round_t round); - -/* Convert a timestamp to a number of seconds (secs) and microseconds (us). - us is always positive. This function is similar to _PyTime_AsTimeval() - except that secs is always a time_t type, whereas the timeval structure - uses a C long for tv_sec on Windows. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( - _PyTime_t t, - time_t *secs, - int *us, - _PyTime_round_t round); - -#if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) -/* Create a timestamp from a timespec structure. - Raise an exception and return -1 on overflow, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts); - -/* Convert a timestamp to a timespec structure (nanosecond resolution). - tv_nsec is always positive. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); - -/* Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. - On overflow, clamp tv_sec to _PyTime_t min/max. */ -PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); -#endif - - -// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -PyAPI_FUNC(_PyTime_t) _PyTime_Add(_PyTime_t t1, _PyTime_t t2); - -/* Compute ticks * mul / div. - Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. - The caller must ensure that ((div - 1) * mul) cannot overflow. */ -PyAPI_FUNC(_PyTime_t) _PyTime_MulDiv(_PyTime_t ticks, - _PyTime_t mul, - _PyTime_t div); - -/* Structure used by time.get_clock_info() */ -typedef struct { - const char *implementation; - int monotonic; - int adjustable; - double resolution; -} _Py_clock_info_t; - -/* Get the current time from the system clock. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetSystemClockWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); - -/* Get the current time from the system clock. - * On success, set *t and *info (if not NULL), and return 0. - * On error, raise an exception and return -1. - */ -PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetMonotonicClockWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - - -/* Converts a timestamp to the Gregorian time, using the local time zone. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); - -/* Converts a timestamp to the Gregorian time, assuming UTC. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); - -/* Get the performance counter: clock with the highest available resolution to - measure a short duration. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetPerfCounterWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); - -/* Get the performance counter: clock with the highest available resolution to - measure a short duration. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_GetPerfCounterWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - - -// Create a deadline. -// Pseudo code: _PyTime_GetMonotonicClock() + timeout. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); - -// Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_GetMonotonicClock(). -PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); - -#ifdef __cplusplus -} -#endif - -#endif /* Py_PYTIME_H */ -#endif /* Py_LIMITED_API */ diff --git a/Include/cpython/setobject.h b/Include/cpython/setobject.h index 20fd63eaae56e2..1778c778a05324 100644 --- a/Include/cpython/setobject.h +++ b/Include/cpython/setobject.h @@ -65,8 +65,3 @@ static inline Py_ssize_t PySet_GET_SIZE(PyObject *so) { return _PySet_CAST(so)->used; } #define PySet_GET_SIZE(so) PySet_GET_SIZE(_PyObject_CAST(so)) - -PyAPI_DATA(PyObject *) _PySet_Dummy; - -PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); -PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h index 19d9dddc344a4f..a3ac07f538a94f 100644 --- a/Include/cpython/sysmodule.h +++ b/Include/cpython/sysmodule.h @@ -2,15 +2,21 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(PyObject *) _PySys_GetAttr(PyThreadState *tstate, - PyObject *name); - -PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); - typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); -PyAPI_FUNC(int) PySys_Audit( - const char *event, - const char *argFormat, - ...); PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; +} PerfMapState; + +PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); +PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( + const void *code_addr, + unsigned int code_size, + const char *entry_name); +PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); +PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index a4e087b2b4eced..81c51944f136f2 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -11,6 +11,3 @@ struct _traceback { int tb_lasti; int tb_lineno; }; - -PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, PyObject **); -PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); diff --git a/Include/cpython/tracemalloc.h b/Include/cpython/tracemalloc.h new file mode 100644 index 00000000000000..61a16ea9a9f3eb --- /dev/null +++ b/Include/cpython/tracemalloc.h @@ -0,0 +1,26 @@ +#ifndef Py_LIMITED_API +#ifndef Py_TRACEMALLOC_H +#define Py_TRACEMALLOC_H + +/* Track an allocated memory block in the tracemalloc module. + Return 0 on success, return -1 on error (failed to allocate memory to store + the trace). + + Return -2 if tracemalloc is disabled. + + If memory block is already tracked, update the existing trace. */ +PyAPI_FUNC(int) PyTraceMalloc_Track( + unsigned int domain, + uintptr_t ptr, + size_t size); + +/* Untrack an allocated memory block in the tracemalloc module. + Do nothing if the block was not tracked. + + Return -2 if tracemalloc is disabled, otherwise return 0. */ +PyAPI_FUNC(int) PyTraceMalloc_Untrack( + unsigned int domain, + uintptr_t ptr); + +#endif // !Py_TRACEMALLOC_H +#endif // !Py_LIMITED_API diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index f6a1f076e03330..e530c8beda44ab 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -11,7 +11,6 @@ typedef struct { } PyTupleObject; PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); /* Cast argument to PyTupleObject* type. */ #define _PyTuple_CAST(op) \ @@ -31,9 +30,9 @@ static inline Py_ssize_t PyTuple_GET_SIZE(PyObject *op) { static inline void PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { PyTupleObject *tuple = _PyTuple_CAST(op); + assert(0 <= index); + assert(index < Py_SIZE(tuple)); tuple->ob_item[index] = value; } #define PyTuple_SET_ITEM(op, index, value) \ PyTuple_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) - -PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 3394726dfffd72..d9b54bce83202d 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -6,8 +6,9 @@ Python and represents a single Unicode element in the Unicode type. With PEP 393, Py_UNICODE is deprecated and replaced with a typedef to wchar_t. */ -#define PY_UNICODE_TYPE wchar_t -/* Py_DEPRECATED(3.3) */ typedef wchar_t Py_UNICODE; +Py_DEPRECATED(3.13) typedef wchar_t PY_UNICODE_TYPE; +Py_DEPRECATED(3.13) typedef wchar_t Py_UNICODE; + /* --- Internal Unicode Operations ---------------------------------------- */ @@ -43,6 +44,7 @@ static inline Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) { return (0xDC00 + (ch & 0x3FF)); } + /* --- Unicode Type ------------------------------------------------------- */ /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject @@ -140,9 +142,11 @@ typedef struct { and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is set, use the PyASCIIObject structure. */ unsigned int ascii:1; + /* The object is statically allocated. */ + unsigned int statically_allocated:1; /* Padding to ensure that PyUnicode_DATA() is always aligned to 4 bytes (see issue #19537 on m68k). */ - unsigned int :25; + unsigned int :24; } state; } PyASCIIObject; @@ -167,10 +171,6 @@ typedef struct { } data; /* Canonical, smallest-form Unicode buffer */ } PyUnicodeObject; -PyAPI_FUNC(int) _PyUnicode_CheckConsistency( - PyObject *op, - int check_content); - #define _PyASCIIObject_CAST(op) \ (assert(PyUnicode_Check(op)), \ @@ -377,9 +377,8 @@ static inline Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *op) #define PyUnicode_MAX_CHAR_VALUE(op) \ PyUnicode_MAX_CHAR_VALUE(_PyObject_CAST(op)) -/* === Public API ========================================================= */ -/* --- Plain Py_UNICODE --------------------------------------------------- */ +/* === Public API ========================================================= */ /* With PEP 393, this is the recommended way to allocate a new unicode object. This function will allocate the object and its buffer in a single memory @@ -396,11 +395,6 @@ static inline int PyUnicode_READY(PyObject* Py_UNUSED(op)) } #define PyUnicode_READY(op) PyUnicode_READY(_PyObject_CAST(op)) -/* Get a copy of a Unicode string. */ -PyAPI_FUNC(PyObject*) _PyUnicode_Copy( - PyObject *unicode - ); - /* Copy character from one unicode object into another, this function performs character conversion when necessary and falls back to memcpy() if possible. @@ -427,17 +421,6 @@ PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters( Py_ssize_t how_many ); -/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so - may crash if parameters are invalid (e.g. if the output string - is too short). */ -PyAPI_FUNC(void) _PyUnicode_FastCopyCharacters( - PyObject *to, - Py_ssize_t to_start, - PyObject *from, - Py_ssize_t from_start, - Py_ssize_t how_many - ); - /* Fill a string with a character: write fill_char into unicode[start:start+length]. @@ -453,15 +436,6 @@ PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill( Py_UCS4 fill_char ); -/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash - if parameters are invalid (e.g. if length is longer than the string). */ -PyAPI_FUNC(void) _PyUnicode_FastFill( - PyObject *unicode, - Py_ssize_t start, - Py_ssize_t length, - Py_UCS4 fill_char - ); - /* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters. Scan the string to find the maximum character. */ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( @@ -469,18 +443,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( const void *buffer, Py_ssize_t size); -/* Create a new string from a buffer of ASCII characters. - WARNING: Don't check if the string contains any non-ASCII character. */ -PyAPI_FUNC(PyObject*) _PyUnicode_FromASCII( - const char *buffer, - Py_ssize_t size); - -/* Compute the maximum character of the substring unicode[start:end]. - Return 127 for an empty string. */ -PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar ( - PyObject *unicode, - Py_ssize_t start, - Py_ssize_t end); /* --- _PyUnicodeWriter API ----------------------------------------------- */ @@ -506,11 +468,11 @@ typedef struct { unsigned char readonly; } _PyUnicodeWriter ; -/* Initialize a Unicode writer. - * - * By default, the minimum buffer size is 0 character and overallocation is - * disabled. Set min_length, min_char and overallocate attributes to control - * the allocation of the buffer. */ +// Initialize a Unicode writer. +// +// By default, the minimum buffer size is 0 character and overallocation is +// disabled. Set min_length, min_char and overallocate attributes to control +// the allocation of the buffer. PyAPI_FUNC(void) _PyUnicodeWriter_Init(_PyUnicodeWriter *writer); @@ -598,15 +560,6 @@ PyAPI_FUNC(void) _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); - /* --- Manage the default encoding ---------------------------------------- */ /* Returns a pointer to the default encoding (UTF-8) of the @@ -624,171 +577,9 @@ PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); +// Alias kept for backward compatibility #define _PyUnicode_AsString PyUnicode_AsUTF8 -/* --- UTF-7 Codecs ------------------------------------------------------- */ - -PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF7( - PyObject *unicode, /* Unicode object */ - int base64SetO, /* Encode RFC2152 Set O characters in base64 */ - int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ - const char *errors /* error handling */ - ); - -/* --- UTF-8 Codecs ------------------------------------------------------- */ - -PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String( - PyObject *unicode, - const char *errors); - -/* --- UTF-32 Codecs ------------------------------------------------------ */ - -PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32( - PyObject *object, /* Unicode object */ - const char *errors, /* error handling */ - int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ - ); - -/* --- UTF-16 Codecs ------------------------------------------------------ */ - -/* Returns a Python string object holding the UTF-16 encoded value of - the Unicode data. - - If byteorder is not 0, output is written according to the following - byte order: - - byteorder == -1: little endian - byteorder == 0: native byte order (writes a BOM mark) - byteorder == 1: big endian - - If byteorder is 0, the output string will always start with the - Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is - prepended. -*/ -PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( - PyObject* unicode, /* Unicode object */ - const char *errors, /* error handling */ - int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ - ); - -/* --- Unicode-Escape Codecs ---------------------------------------------- */ - -/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeStateful( - const char *string, /* Unicode-Escape encoded string */ - Py_ssize_t length, /* size of string */ - const char *errors, /* error handling */ - Py_ssize_t *consumed /* bytes consumed */ -); -/* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape - chars. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( - const char *string, /* Unicode-Escape encoded string */ - Py_ssize_t length, /* size of string */ - const char *errors, /* error handling */ - Py_ssize_t *consumed, /* bytes consumed */ - const char **first_invalid_escape /* on return, points to first - invalid escaped char in - string. */ -); - -/* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */ - -/* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeRawUnicodeEscapeStateful( - const char *string, /* Unicode-Escape encoded string */ - Py_ssize_t length, /* size of string */ - const char *errors, /* error handling */ - Py_ssize_t *consumed /* bytes consumed */ -); - -/* --- Latin-1 Codecs ----------------------------------------------------- */ - -PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String( - PyObject* unicode, - const char* errors); - -/* --- ASCII Codecs ------------------------------------------------------- */ - -PyAPI_FUNC(PyObject*) _PyUnicode_AsASCIIString( - PyObject* unicode, - const char* errors); - -/* --- Character Map Codecs ----------------------------------------------- */ - -/* Translate an Unicode object by applying a character mapping table to - it and return the resulting Unicode object. - - The mapping table must map Unicode ordinal integers to Unicode strings, - Unicode ordinal integers or None (causing deletion of the character). - - Mapping tables may be dictionaries or sequences. Unmapped character - ordinals (ones which cause a LookupError) are left untouched and - are copied as-is. -*/ -PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( - PyObject *unicode, /* Unicode object */ - PyObject *mapping, /* encoding mapping */ - const char *errors /* error handling */ - ); - -/* --- Decimal Encoder ---------------------------------------------------- */ - -/* Coverts a Unicode object holding a decimal value to an ASCII string - for using in int, float and complex parsers. - Transforms code points that have decimal digit property to the - corresponding ASCII digit code points. Transforms spaces to ASCII. - Transforms code points starting from the first non-ASCII code point that - is neither a decimal digit nor a space to the end into '?'. */ - -PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( - PyObject *unicode /* Unicode object */ - ); - -/* --- Methods & Slots ---------------------------------------------------- */ - -PyAPI_FUNC(PyObject *) _PyUnicode_JoinArray( - PyObject *separator, - PyObject *const *items, - Py_ssize_t seqlen - ); - -/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, - 0 otherwise. The right argument must be ASCII identifier. - Any error occurs inside will be cleared before return. */ -PyAPI_FUNC(int) _PyUnicode_EqualToASCIIId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); - -/* Test whether a unicode is equal to ASCII string. Return 1 if true, - 0 otherwise. The right argument must be ASCII-encoded string. - Any error occurs inside will be cleared before return. */ -PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString( - PyObject *left, - const char *right /* ASCII-encoded string */ - ); - -/* Externally visible for str.strip(unicode) */ -PyAPI_FUNC(PyObject *) _PyUnicode_XStrip( - PyObject *self, - int striptype, - PyObject *sepobj - ); - -/* Using explicit passed-in values, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ -PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar); /* === Characters Type APIs =============================================== */ @@ -811,14 +602,6 @@ PyAPI_FUNC(int) _PyUnicode_IsTitlecase( Py_UCS4 ch /* Unicode character */ ); -PyAPI_FUNC(int) _PyUnicode_IsXidStart( - Py_UCS4 ch /* Unicode character */ - ); - -PyAPI_FUNC(int) _PyUnicode_IsXidContinue( - Py_UCS4 ch /* Unicode character */ - ); - PyAPI_FUNC(int) _PyUnicode_IsWhitespace( const Py_UCS4 ch /* Unicode character */ ); @@ -827,46 +610,18 @@ PyAPI_FUNC(int) _PyUnicode_IsLinebreak( const Py_UCS4 ch /* Unicode character */ ); -/* Py_DEPRECATED(3.3) */ PyAPI_FUNC(Py_UCS4) _PyUnicode_ToLowercase( +PyAPI_FUNC(Py_UCS4) _PyUnicode_ToLowercase( Py_UCS4 ch /* Unicode character */ ); -/* Py_DEPRECATED(3.3) */ PyAPI_FUNC(Py_UCS4) _PyUnicode_ToUppercase( +PyAPI_FUNC(Py_UCS4) _PyUnicode_ToUppercase( Py_UCS4 ch /* Unicode character */ ); -Py_DEPRECATED(3.3) PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase( +PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase( Py_UCS4 ch /* Unicode character */ ); -PyAPI_FUNC(int) _PyUnicode_ToLowerFull( - Py_UCS4 ch, /* Unicode character */ - Py_UCS4 *res - ); - -PyAPI_FUNC(int) _PyUnicode_ToTitleFull( - Py_UCS4 ch, /* Unicode character */ - Py_UCS4 *res - ); - -PyAPI_FUNC(int) _PyUnicode_ToUpperFull( - Py_UCS4 ch, /* Unicode character */ - Py_UCS4 *res - ); - -PyAPI_FUNC(int) _PyUnicode_ToFoldedFull( - Py_UCS4 ch, /* Unicode character */ - Py_UCS4 *res - ); - -PyAPI_FUNC(int) _PyUnicode_IsCaseIgnorable( - Py_UCS4 ch /* Unicode character */ - ); - -PyAPI_FUNC(int) _PyUnicode_IsCased( - Py_UCS4 ch /* Unicode character */ - ); - PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit( Py_UCS4 ch /* Unicode character */ ); @@ -943,19 +698,6 @@ static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) { /* === Misc functions ===================================================== */ -PyAPI_FUNC(PyObject*) _PyUnicode_FormatLong(PyObject *, int, int, int); - -/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/ +// Return an interned Unicode object for an Identifier; may fail if there is no +// memory. PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); - -/* Fast equality check when the inputs are known to be exact unicode types - and where the hash values are equal (i.e. a very probable match) */ -PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *); - -/* Equality check. */ -PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *); - -PyAPI_FUNC(int) _PyUnicode_WideCharString_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *); - -PyAPI_FUNC(Py_ssize_t) _PyUnicode_ScanIdentifier(PyObject *); diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index fd79fdc2dcc468..1559e2def61260 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -32,11 +32,8 @@ struct _PyWeakReference { vectorcallfunc vectorcall; }; -PyAPI_FUNC(Py_ssize_t) _PyWeakref_GetWeakrefCount(PyWeakReference *head); - -PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); - -static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { +Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) +{ PyWeakReference *ref; PyObject *obj; assert(PyWeakref_Check(ref_obj)); diff --git a/Include/dictobject.h b/Include/dictobject.h index e7fcb44d0cf9a9..1bbeec1ab699e7 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -57,6 +57,17 @@ PyAPI_FUNC(int) PyDict_MergeFromSeq2(PyObject *d, PyAPI_FUNC(PyObject *) PyDict_GetItemString(PyObject *dp, const char *key); PyAPI_FUNC(int) PyDict_SetItemString(PyObject *dp, const char *key, PyObject *item); PyAPI_FUNC(int) PyDict_DelItemString(PyObject *dp, const char *key); + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 +// Return the object from dictionary *op* which has a key *key*. +// - If the key is present, set *result to a new strong reference to the value +// and return 1. +// - If the key is missing, set *result to NULL and return 0 . +// - On error, raise an exception and return -1. +PyAPI_FUNC(int) PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result); +PyAPI_FUNC(int) PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result); +#endif + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); #endif diff --git a/Include/errcode.h b/Include/errcode.h index 54ae929bf25870..dac5cf068c99d6 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -1,36 +1,43 @@ +// Error codes passed around between file input, tokenizer, parser and +// interpreter. This is necessary so we can turn them into Python +// exceptions at a higher level. Note that some errors have a +// slightly different meaning when passed from the tokenizer to the +// parser than when passed from the parser to the interpreter; e.g. +// the parser only returns E_EOF when it hits EOF immediately, and it +// never returns E_OK. +// +// The public PyRun_InteractiveOneObjectEx() function can return E_EOF, +// same as its variants: +// +// * PyRun_InteractiveOneObject() +// * PyRun_InteractiveOneFlags() +// * PyRun_InteractiveOne() + #ifndef Py_ERRCODE_H #define Py_ERRCODE_H #ifdef __cplusplus extern "C" { #endif - -/* Error codes passed around between file input, tokenizer, parser and - interpreter. This is necessary so we can turn them into Python - exceptions at a higher level. Note that some errors have a - slightly different meaning when passed from the tokenizer to the - parser than when passed from the parser to the interpreter; e.g. - the parser only returns E_EOF when it hits EOF immediately, and it - never returns E_OK. */ - -#define E_OK 10 /* No error */ -#define E_EOF 11 /* End Of File */ -#define E_INTR 12 /* Interrupted */ -#define E_TOKEN 13 /* Bad token */ -#define E_SYNTAX 14 /* Syntax error */ -#define E_NOMEM 15 /* Ran out of memory */ -#define E_DONE 16 /* Parsing complete */ -#define E_ERROR 17 /* Execution error */ -#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */ -#define E_OVERFLOW 19 /* Node had too many children */ -#define E_TOODEEP 20 /* Too many indentation levels */ -#define E_DEDENT 21 /* No matching outer block for dedent */ -#define E_DECODE 22 /* Error in decoding into Unicode */ -#define E_EOFS 23 /* EOF in triple-quoted string */ -#define E_EOLS 24 /* EOL in single-quoted string */ -#define E_LINECONT 25 /* Unexpected characters after a line continuation */ -#define E_BADSINGLE 27 /* Ill-formed single statement input */ -#define E_INTERACT_STOP 28 /* Interactive mode stopped tokenization */ +#define E_OK 10 /* No error */ +#define E_EOF 11 /* End Of File */ +#define E_INTR 12 /* Interrupted */ +#define E_TOKEN 13 /* Bad token */ +#define E_SYNTAX 14 /* Syntax error */ +#define E_NOMEM 15 /* Ran out of memory */ +#define E_DONE 16 /* Parsing complete */ +#define E_ERROR 17 /* Execution error */ +#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */ +#define E_OVERFLOW 19 /* Node had too many children */ +#define E_TOODEEP 20 /* Too many indentation levels */ +#define E_DEDENT 21 /* No matching outer block for dedent */ +#define E_DECODE 22 /* Error in decoding into Unicode */ +#define E_EOFS 23 /* EOF in triple-quoted string */ +#define E_EOLS 24 /* EOL in single-quoted string */ +#define E_LINECONT 25 /* Unexpected characters after a line continuation */ +#define E_BADSINGLE 27 /* Ill-formed single statement input */ +#define E_INTERACT_STOP 28 /* Interactive mode stopped tokenization */ +#define E_COLUMNOVERFLOW 29 /* Column offset overflow */ #ifdef __cplusplus } diff --git a/Include/exports.h b/Include/exports.h index 59373c39ff757c..ce601216f17156 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -1,6 +1,29 @@ #ifndef Py_EXPORTS_H #define Py_EXPORTS_H +/* Declarations for symbol visibility. + + PyAPI_FUNC(type): Declares a public Python API function and return type + PyAPI_DATA(type): Declares public Python data and its type + PyMODINIT_FUNC: A Python module init function. If these functions are + inside the Python core, they are private to the core. + If in an extension module, it may be declared with + external linkage depending on the platform. + + As a number of platforms support/require "__declspec(dllimport/dllexport)", + we support a HAVE_DECLSPEC_DLL macro to save duplication. +*/ + +/* + All windows ports, except cygwin, are handled in PC/pyconfig.h. + + Cygwin is the only other autoconf platform requiring special + linkage handling and it uses __declspec(). +*/ +#if defined(__CYGWIN__) +# define HAVE_DECLSPEC_DLL +#endif + #if defined(_WIN32) || defined(__CYGWIN__) #if defined(Py_ENABLE_SHARED) #define Py_IMPORTED_SYMBOL __declspec(dllimport) @@ -33,4 +56,53 @@ #endif #endif +/* only get special linkage if built as shared or platform is Cygwin */ +#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) +# if defined(HAVE_DECLSPEC_DLL) +# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE +# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE + /* module init functions inside the core need no external linkage */ + /* except for Cygwin to handle embedding */ +# if defined(__CYGWIN__) +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# else /* __CYGWIN__ */ +# define PyMODINIT_FUNC PyObject* +# endif /* __CYGWIN__ */ +# else /* Py_BUILD_CORE */ + /* Building an extension module, or an embedded situation */ + /* public Python functions and data are imported */ + /* Under Cygwin, auto-import functions to prevent compilation */ + /* failures similar to those described at the bottom of 4.1: */ + /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ +# if !defined(__CYGWIN__) +# define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE +# endif /* !__CYGWIN__ */ +# define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE + /* module init functions outside the core must be exported */ +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# endif /* __cplusplus */ +# endif /* Py_BUILD_CORE */ +# endif /* HAVE_DECLSPEC_DLL */ +#endif /* Py_ENABLE_SHARED */ + +/* If no external linkage macros defined by now, create defaults */ +#ifndef PyAPI_FUNC +# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE +#endif +#ifndef PyAPI_DATA +# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE +#endif +#ifndef PyMODINIT_FUNC +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# endif /* __cplusplus */ +#endif + + #endif /* Py_EXPORTS_H */ diff --git a/Include/fileobject.h b/Include/fileobject.h index 02bd7c915a23f7..6a6d11409497fa 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -23,20 +23,12 @@ Py_DEPRECATED(3.12) PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 Py_DEPRECATED(3.12) PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; #endif -PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; +Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_UTF8Mode; #endif -/* A routine to check if a file descriptor can be select()-ed. */ -#ifdef _MSC_VER - /* On Windows, any socket fd can be select()-ed, no matter how high */ - #define _PyIsSelectable_fd(FD) (1) -#else - #define _PyIsSelectable_fd(FD) ((unsigned int)(FD) < (unsigned int)FD_SETSIZE) -#endif - #ifndef Py_LIMITED_API # define Py_CPYTHON_FILEOBJECT_H # include "cpython/fileobject.h" diff --git a/Include/fileutils.h b/Include/fileutils.h index ba5acc84fcb185..1509198e45f0ca 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -1,5 +1,41 @@ #ifndef Py_FILEUTILS_H #define Py_FILEUTILS_H + +/******************************* + * stat() and fstat() fiddling * + *******************************/ + +#ifdef HAVE_SYS_STAT_H +# include // S_ISREG() +#elif defined(HAVE_STAT_H) +# include // S_ISREG() +#endif + +#ifndef S_IFMT + // VisualAge C/C++ Failed to Define MountType Field in sys/stat.h. +# define S_IFMT 0170000 +#endif +#ifndef S_IFLNK + // Windows doesn't define S_IFLNK, but posixmodule.c maps + // IO_REPARSE_TAG_SYMLINK to S_IFLNK. +# define S_IFLNK 0120000 +#endif +#ifndef S_ISREG +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif +#ifndef S_ISDIR +# define S_ISDIR(x) (((x) & S_IFMT) == S_IFDIR) +#endif +#ifndef S_ISCHR +# define S_ISCHR(x) (((x) & S_IFMT) == S_IFCHR) +#endif +#ifndef S_ISLNK +# define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) +#endif + + +// Move this down here since some C++ #include's don't like to be included +// inside an extern "C". #ifdef __cplusplus extern "C" { #endif diff --git a/Include/import.h b/Include/import.h index 5d5f3425b8e715..24b23b9119196f 100644 --- a/Include/import.h +++ b/Include/import.h @@ -43,10 +43,15 @@ PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( PyAPI_FUNC(PyObject *) PyImport_AddModule( const char *name /* UTF-8 encoded string */ ); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject *) PyImport_AddModuleRef( + const char *name /* UTF-8 encoded string */ + ); +#endif PyAPI_FUNC(PyObject *) PyImport_ImportModule( const char *name /* UTF-8 encoded string */ ); -PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock( +Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock( const char *name /* UTF-8 encoded string */ ); PyAPI_FUNC(PyObject *) PyImport_ImportModuleLevel( diff --git a/Include/internal/mimalloc/mimalloc.h b/Include/internal/mimalloc/mimalloc.h new file mode 100644 index 00000000000000..821129e7690b1b --- /dev/null +++ b/Include/internal/mimalloc/mimalloc.h @@ -0,0 +1,565 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_H +#define MIMALLOC_H + +#define MI_MALLOC_VERSION 212 // major + 2 digits minor + +// ------------------------------------------------------ +// Compiler specific attributes +// ------------------------------------------------------ + +#ifdef __cplusplus + #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 + #define mi_attr_noexcept noexcept + #else + #define mi_attr_noexcept throw() + #endif +#else + #define mi_attr_noexcept +#endif + +#if defined(__cplusplus) && (__cplusplus >= 201703) + #define mi_decl_nodiscard [[nodiscard]] +#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl + #define mi_decl_nodiscard __attribute__((warn_unused_result)) +#elif defined(_HAS_NODISCARD) + #define mi_decl_nodiscard _NODISCARD +#elif (_MSC_VER >= 1700) + #define mi_decl_nodiscard _Check_return_ +#else + #define mi_decl_nodiscard +#endif + +#if defined(_MSC_VER) || defined(__MINGW32__) + #if !defined(MI_SHARED_LIB) + #define mi_decl_export + #elif defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __declspec(dllexport) + #else + #define mi_decl_export __declspec(dllimport) + #endif + #if defined(__MINGW32__) + #define mi_decl_restrict + #define mi_attr_malloc __attribute__((malloc)) + #else + #if (_MSC_VER >= 1900) && !defined(__EDG__) + #define mi_decl_restrict __declspec(allocator) __declspec(restrict) + #else + #define mi_decl_restrict __declspec(restrict) + #endif + #define mi_attr_malloc + #endif + #define mi_cdecl __cdecl + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) +#elif defined(__GNUC__) // includes clang and icc + #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __attribute__((visibility("default"))) + #else + #define mi_decl_export + #endif + #define mi_cdecl // leads to warnings... __attribute__((cdecl)) + #define mi_decl_restrict + #define mi_attr_malloc __attribute__((malloc)) + #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) + #elif defined(__INTEL_COMPILER) + #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_attr_alloc_align(p) + #else + #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_attr_alloc_align(p) __attribute__((alloc_align(p))) + #endif +#else + #define mi_cdecl + #define mi_decl_export + #define mi_decl_restrict + #define mi_attr_malloc + #define mi_attr_alloc_size(s) + #define mi_attr_alloc_size2(s1,s2) + #define mi_attr_alloc_align(p) +#endif + +// ------------------------------------------------------ +// Includes +// ------------------------------------------------------ + +#include // size_t +#include // bool +#include // INTPTR_MAX + +#ifdef __cplusplus +extern "C" { +#endif + +// ------------------------------------------------------ +// Standard malloc interface +// ------------------------------------------------------ + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); + +mi_decl_export void mi_free(void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; + +// ------------------------------------------------------ +// Extended functionality +// ------------------------------------------------------ +#define MI_SMALL_WSIZE_MAX (128) +#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); + +mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; + + +// ------------------------------------------------------ +// Internals +// ------------------------------------------------------ + +typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg); +mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept; + +typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg); +mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept; + +typedef void (mi_cdecl mi_error_fun)(int err, void* arg); +mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg); + +mi_decl_export void mi_collect(bool force) mi_attr_noexcept; +mi_decl_export int mi_version(void) mi_attr_noexcept; +mi_decl_export void mi_stats_reset(void) mi_attr_noexcept; +mi_decl_export void mi_stats_merge(void) mi_attr_noexcept; +mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL +mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; + +mi_decl_export void mi_process_init(void) mi_attr_noexcept; +mi_decl_export void mi_thread_init(void) mi_attr_noexcept; +mi_decl_export void mi_thread_done(void) mi_attr_noexcept; +mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; + +mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, + size_t* current_rss, size_t* peak_rss, + size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept; + +// ------------------------------------------------------------------------------------- +// Aligned allocation +// Note that `alignment` always follows `size` for consistency with unaligned +// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. +// ------------------------------------------------------------------------------------- + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); + + +// ------------------------------------------------------------------------------------- +// Heaps: first-class, but can only allocate from the same thread that created it. +// ------------------------------------------------------------------------------------- + +struct mi_heap_s; +typedef struct mi_heap_s mi_heap_t; + +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void); +mi_decl_export void mi_heap_delete(mi_heap_t* heap); +mi_decl_export void mi_heap_destroy(mi_heap_t* heap); +mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap); +mi_decl_export mi_heap_t* mi_heap_get_default(void); +mi_decl_export mi_heap_t* mi_heap_get_backing(void); +mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); + +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); +mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); + + +// -------------------------------------------------------------------------------- +// Zero initialized re-allocation. +// Only valid on memory that was originally allocated with zero initialization too. +// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc. +// see +// -------------------------------------------------------------------------------- + +mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); + +mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3); + +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); + +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); +mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4); + + +// ------------------------------------------------------ +// Analysis +// ------------------------------------------------------ + +mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p); +mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p); +mi_decl_export bool mi_check_owned(const void* p); + +// An area of heap space contains blocks of a single size. +typedef struct mi_heap_area_s { + void* blocks; // start of the area containing heap blocks + size_t reserved; // bytes reserved for this area (virtual) + size_t committed; // current available bytes for this area + size_t used; // number of allocated blocks + size_t block_size; // size in bytes of each block + size_t full_block_size; // size in bytes of a full block including padding and metadata. +} mi_heap_area_t; + +typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); + +mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); + +// Experimental +mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept; + +mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept; +mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept; + +mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; +mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; + +mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept; + +// Experimental: heaps associated with specific memory arena's +typedef int mi_arena_id_t; +mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); +mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; +mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; + +#if MI_MALLOC_VERSION >= 182 +// Create a heap that only allocates in the specified arena +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); +#endif + +// deprecated +mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; + + +// ------------------------------------------------------ +// Convenience +// ------------------------------------------------------ + +#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) +#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) +#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) +#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp))) +#define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp))) +#define mi_recalloc_tp(p,tp,n) ((tp*)mi_recalloc(p,n,sizeof(tp))) + +#define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) +#define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) +#define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp))) +#define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp))) +#define mi_heap_reallocn_tp(hp,p,tp,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp))) +#define mi_heap_recalloc_tp(hp,p,tp,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp))) + + +// ------------------------------------------------------ +// Options +// ------------------------------------------------------ + +typedef enum mi_option_e { + // stable options + mi_option_show_errors, // print error messages + mi_option_show_stats, // print statistics on termination + mi_option_verbose, // print verbose messages + // the following options are experimental (see src/options.h) + mi_option_eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1) + mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2) + mi_option_purge_decommits, // should a memory purge decommit (or only reset) (=1) + mi_option_allow_large_os_pages, // allow large (2MiB) OS pages, implies eager commit + mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB/page) at startup + mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node + mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup + mi_option_deprecated_segment_cache, + mi_option_deprecated_page_reset, + mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination + mi_option_deprecated_segment_reset, + mi_option_eager_commit_delay, + mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. + mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. + mi_option_limit_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas) + mi_option_os_tag, // tag used for OS logging (macOS only for now) + mi_option_max_errors, // issue at most N error messages + mi_option_max_warnings, // issue at most N warning messages + mi_option_max_segment_reclaim, + mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe. + mi_option_arena_reserve, // initial memory size in KiB for arena reservation (1GiB on 64-bit) + mi_option_arena_purge_mult, + mi_option_purge_extend_delay, + _mi_option_last, + // legacy option names + mi_option_large_os_pages = mi_option_allow_large_os_pages, + mi_option_eager_region_commit = mi_option_arena_eager_commit, + mi_option_reset_decommits = mi_option_purge_decommits, + mi_option_reset_delay = mi_option_purge_delay, + mi_option_abandoned_page_reset = mi_option_abandoned_page_purge +} mi_option_t; + + +mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option); +mi_decl_export void mi_option_enable(mi_option_t option); +mi_decl_export void mi_option_disable(mi_option_t option); +mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); +mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); + +mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); +mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); +mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option); +mi_decl_export void mi_option_set(mi_option_t option, long value); +mi_decl_export void mi_option_set_default(mi_option_t option, long value); + + +// ------------------------------------------------------------------------------------------------------- +// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. +// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) +// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing. +// ------------------------------------------------------------------------------------------------------- + +mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; +mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; + +mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); + +mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; +mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept; + +mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; +mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; + +// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`. +// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception). +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2); +mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); + +#ifdef __cplusplus +} +#endif + +// --------------------------------------------------------------------------------------------- +// Implement the C++ std::allocator interface for use in STL containers. +// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally) +// --------------------------------------------------------------------------------------------- +#ifdef __cplusplus + +#include // std::size_t +#include // PTRDIFF_MAX +#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 +#include // std::true_type +#include // std::forward +#endif + +template struct _mi_stl_allocator_common { + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type& reference; + typedef value_type const& const_reference; + typedef value_type* pointer; + typedef value_type const* const_pointer; + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + template void construct(U* p, Args&& ...args) { ::new(p) U(std::forward(args)...); } + template void destroy(U* p) mi_attr_noexcept { p->~U(); } + #else + void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } + void destroy(pointer p) { p->~value_type(); } + #endif + + size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +}; + +template struct mi_stl_allocator : public _mi_stl_allocator_common { + using typename _mi_stl_allocator_common::size_type; + using typename _mi_stl_allocator_common::value_type; + using typename _mi_stl_allocator_common::pointer; + template struct rebind { typedef mi_stl_allocator other; }; + + mi_stl_allocator() mi_attr_noexcept = default; + mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default; + template mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept { } + mi_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T* p, size_type) { mi_free(p); } + + #if (__cplusplus >= 201703L) // C++17 + mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_new_n(count, sizeof(T))); } + mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } + #else + mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_new_n(count, sizeof(value_type))); } + #endif + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using is_always_equal = std::true_type; + #endif +}; + +template bool operator==(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return true; } +template bool operator!=(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return false; } + + +#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11 +#define MI_HAS_HEAP_STL_ALLOCATOR 1 + +#include // std::shared_ptr + +// Common base class for STL allocators in a specific heap +template struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common { + using typename _mi_stl_allocator_common::size_type; + using typename _mi_stl_allocator_common::value_type; + using typename _mi_stl_allocator_common::pointer; + + _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp) { } /* will not delete nor destroy the passed in heap */ + + #if (__cplusplus >= 201703L) // C++17 + mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); } + mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } + #else + mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); } + #endif + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using is_always_equal = std::false_type; + #endif + + void collect(bool force) { mi_heap_collect(this->heap.get(), force); } + template bool is_equal(const _mi_heap_stl_allocator_common& x) const { return (this->heap == x.heap); } + +protected: + std::shared_ptr heap; + template friend struct _mi_heap_stl_allocator_common; + + _mi_heap_stl_allocator_common() { + mi_heap_t* hp = mi_heap_new(); + this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ + } + _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } + template _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } + +private: + static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } } + static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } } +}; + +// STL allocator allocation in a specific heap +template struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common { + using typename _mi_heap_stl_allocator_common::size_type; + mi_heap_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is deleted when the destructor is called + mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap + template mi_heap_stl_allocator(const mi_heap_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { } + + mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T* p, size_type) { mi_free(p); } + template struct rebind { typedef mi_heap_stl_allocator other; }; +}; + +template bool operator==(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (x.is_equal(y)); } +template bool operator!=(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (!x.is_equal(y)); } + + +// STL allocator allocation in a specific heap, where `free` does nothing and +// the heap is destroyed in one go on destruction -- use with care! +template struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common { + using typename _mi_heap_stl_allocator_common::size_type; + mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is destroyed when the destructor is called + mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap + template mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { } + + mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ } + template struct rebind { typedef mi_heap_destroy_stl_allocator other; }; +}; + +template bool operator==(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (x.is_equal(y)); } +template bool operator!=(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (!x.is_equal(y)); } + +#endif // C++11 + +#endif // __cplusplus + +#endif diff --git a/Include/internal/mimalloc/mimalloc/atomic.h b/Include/internal/mimalloc/mimalloc/atomic.h new file mode 100644 index 00000000000000..eb8478ceed6adf --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/atomic.h @@ -0,0 +1,385 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_ATOMIC_H +#define MIMALLOC_ATOMIC_H + +// -------------------------------------------------------------------------------------------- +// Atomics +// We need to be portable between C, C++, and MSVC. +// We base the primitives on the C/C++ atomics and create a mimimal wrapper for MSVC in C compilation mode. +// This is why we try to use only `uintptr_t` and `*` as atomic types. +// To gain better insight in the range of used atomics, we use explicitly named memory order operations +// instead of passing the memory order as a parameter. +// ----------------------------------------------------------------------------------------------- + +#if defined(__cplusplus) +// Use C++ atomics +#include +#define _Atomic(tp) std::atomic +#define mi_atomic(name) std::atomic_##name +#define mi_memory_order(name) std::memory_order_##name +#if !defined(ATOMIC_VAR_INIT) || (__cplusplus >= 202002L) // c++20, see issue #571 + #define MI_ATOMIC_VAR_INIT(x) x +#else + #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) +#endif +#elif defined(_MSC_VER) +// Use MSVC C wrapper for C11 atomics +#define _Atomic(tp) tp +#define MI_ATOMIC_VAR_INIT(x) x +#define mi_atomic(name) mi_atomic_##name +#define mi_memory_order(name) mi_memory_order_##name +#else +// Use C11 atomics +#include +#define mi_atomic(name) atomic_##name +#define mi_memory_order(name) memory_order_##name +#if !defined(ATOMIC_VAR_INIT) || (__STDC_VERSION__ >= 201710L) // c17, see issue #735 + #define MI_ATOMIC_VAR_INIT(x) x +#else + #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) +#endif +#endif + +// Various defines for all used memory orders in mimalloc +#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \ + mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail) + +#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \ + mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail) + +#define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) +#define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) +#define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) +#define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) + +#define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel)) + +#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1) +#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1) +#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1) +#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1) + +static inline void mi_atomic_yield(void); +static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add); +static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); + + +#if defined(__cplusplus) || !defined(_MSC_VER) + +// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value) +// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well +#define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p) +#define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p) + +// In C++ we need to add casts to help resolve templates if NULL is passed +#if defined(__cplusplus) +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des) +#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x) +#else +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des) +#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x) +#endif + +// These are used by the statistics +static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) { + return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed)); +} +static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { + int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p); + while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ }; +} + +// Used by timers +#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) + +#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d) +#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i) + + +#elif defined(_MSC_VER) + +// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. +#define WIN32_LEAN_AND_MEAN +#include +#include +#ifdef _WIN64 +typedef LONG64 msc_intptr_t; +#define MI_64(f) f##64 +#else +typedef LONG msc_intptr_t; +#define MI_64(f) f +#endif + +typedef enum mi_memory_order_e { + mi_memory_order_relaxed, + mi_memory_order_consume, + mi_memory_order_acquire, + mi_memory_order_release, + mi_memory_order_acq_rel, + mi_memory_order_seq_cst +} mi_memory_order; + +static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add); +} +static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub)); +} +static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x); +} +static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x); +} +static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { + (void)(mo1); (void)(mo2); + uintptr_t read = (uintptr_t)MI_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected)); + if (read == *expected) { + return true; + } + else { + *expected = read; + return false; + } +} +static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { + return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2); +} +static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) { + (void)(mo); + return (uintptr_t)MI_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); +} +static inline void mi_atomic_thread_fence(mi_memory_order mo) { + (void)(mo); + _Atomic(uintptr_t) x = 0; + mi_atomic_exchange_explicit(&x, 1, mo); +} +static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { + (void)(mo); +#if defined(_M_IX86) || defined(_M_X64) + return *p; +#else + uintptr_t x = *p; + if (mo > mi_memory_order_relaxed) { + while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; + } + return x; +#endif +} +static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { + (void)(mo); +#if defined(_M_IX86) || defined(_M_X64) + *p = x; +#else + mi_atomic_exchange_explicit(p, x, mo); +#endif +} +static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) { + (void)(mo); +#if defined(_M_X64) + return *p; +#else + int64_t old = *p; + int64_t x = old; + while ((old = InterlockedCompareExchange64(p, x, old)) != x) { + x = old; + } + return x; +#endif +} +static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) { + (void)(mo); +#if defined(x_M_IX86) || defined(_M_X64) + *p = x; +#else + InterlockedExchange64(p, x); +#endif +} + +// These are used by the statistics +static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) { +#ifdef _WIN64 + return (int64_t)mi_atomic_addi((int64_t*)p, add); +#else + int64_t current; + int64_t sum; + do { + current = *p; + sum = current + add; + } while (_InterlockedCompareExchange64(p, sum, current) != current); + return current; +#endif +} +static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) { + int64_t current; + do { + current = *p; + } while (current < x && _InterlockedCompareExchange64(p, x, current) != current); +} + +static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) { + mi_atomic_addi64_relaxed(p, i); +} + +static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) { + int64_t read = _InterlockedCompareExchange64(p, des, *exp); + if (read == *exp) { + return true; + } + else { + *exp = read; + return false; + } +} + +// The pointer macros cast to `uintptr_t`. +#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p)) +#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p)) +#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) +#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) +#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) +#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x) + +#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire)) +#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed)) +#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed)) + + +#endif + + +// Atomically add a signed value; returns the previous value. +static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) { + return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add); +} + +// Atomically subtract a signed value; returns the previous value. +static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { + return (intptr_t)mi_atomic_addi(p, -sub); +} + +typedef _Atomic(uintptr_t) mi_atomic_once_t; + +// Returns true only on the first invocation +static inline bool mi_atomic_once( mi_atomic_once_t* once ) { + if (mi_atomic_load_relaxed(once) != 0) return false; // quick test + uintptr_t expected = 0; + return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1 +} + +typedef _Atomic(uintptr_t) mi_atomic_guard_t; + +// Allows only one thread to execute at a time +#define mi_atomic_guard(guard) \ + uintptr_t _mi_guard_expected = 0; \ + for(bool _mi_guard_once = true; \ + _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \ + (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) ) + + + +// Yield +#if defined(__cplusplus) +#include +static inline void mi_atomic_yield(void) { + std::this_thread::yield(); +} +#elif defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +static inline void mi_atomic_yield(void) { + YieldProcessor(); +} +#elif defined(__SSE2__) +#include +static inline void mi_atomic_yield(void) { + _mm_pause(); +} +#elif (defined(__GNUC__) || defined(__clang__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__armel__) || defined(__ARMEL__) || \ + defined(__aarch64__) || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)) || defined(__POWERPC__) +#if defined(__x86_64__) || defined(__i386__) +static inline void mi_atomic_yield(void) { + __asm__ volatile ("pause" ::: "memory"); +} +#elif defined(__aarch64__) +static inline void mi_atomic_yield(void) { + __asm__ volatile("wfe"); +} +#elif (defined(__arm__) && __ARM_ARCH__ >= 7) +static inline void mi_atomic_yield(void) { + __asm__ volatile("yield" ::: "memory"); +} +#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__) +#ifdef __APPLE__ +static inline void mi_atomic_yield(void) { + __asm__ volatile ("or r27,r27,r27" ::: "memory"); +} +#else +static inline void mi_atomic_yield(void) { + __asm__ __volatile__ ("or 27,27,27" ::: "memory"); +} +#endif +#elif defined(__armel__) || defined(__ARMEL__) +static inline void mi_atomic_yield(void) { + __asm__ volatile ("nop" ::: "memory"); +} +#endif +#elif defined(__sun) +// Fallback for other archs +#include +static inline void mi_atomic_yield(void) { + smt_pause(); +} +#elif defined(__wasi__) +#include +static inline void mi_atomic_yield(void) { + sched_yield(); +} +#else +#include +static inline void mi_atomic_yield(void) { + sleep(0); +} +#endif + + +#endif // __MIMALLOC_ATOMIC_H diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h new file mode 100644 index 00000000000000..f076bc6a40f977 --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -0,0 +1,979 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_INTERNAL_H +#define MIMALLOC_INTERNAL_H + + +// -------------------------------------------------------------------------- +// This file contains the interal API's of mimalloc and various utility +// functions and macros. +// -------------------------------------------------------------------------- + +#include "mimalloc/types.h" +#include "mimalloc/track.h" + +#if (MI_DEBUG>0) +#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) +#else +#define mi_trace_message(...) +#endif + +#define MI_CACHE_LINE 64 +#if defined(_MSC_VER) +#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) +#pragma warning(disable:26812) // unscoped enum warning +#define mi_decl_noinline __declspec(noinline) +#define mi_decl_thread __declspec(thread) +#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc +#define mi_decl_noinline __attribute__((noinline)) +#define mi_decl_thread __thread +#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) +#else +#define mi_decl_noinline +#define mi_decl_thread __thread // hope for the best :-) +#define mi_decl_cache_align +#endif + +#if defined(__EMSCRIPTEN__) && !defined(__wasi__) +#define __wasi__ +#endif + +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#else +#define mi_decl_externc +#endif + +// pthreads +#if !defined(_WIN32) && !defined(__wasi__) +#define MI_USE_PTHREADS +#include +#endif + +// "options.c" +void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); +void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); +void _mi_warning_message(const char* fmt, ...); +void _mi_verbose_message(const char* fmt, ...); +void _mi_trace_message(const char* fmt, ...); +void _mi_options_init(void); +void _mi_error_message(int err, const char* fmt, ...); + +// random.c +void _mi_random_init(mi_random_ctx_t* ctx); +void _mi_random_init_weak(mi_random_ctx_t* ctx); +void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); +void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); +uintptr_t _mi_random_next(mi_random_ctx_t* ctx); +uintptr_t _mi_heap_random_next(mi_heap_t* heap); +uintptr_t _mi_os_random_weak(uintptr_t extra_seed); +static inline uintptr_t _mi_random_shuffle(uintptr_t x); + +// init.c +extern mi_decl_cache_align mi_stats_t _mi_stats_main; +extern mi_decl_cache_align const mi_page_t _mi_page_empty; +bool _mi_is_main_thread(void); +size_t _mi_current_thread_count(void); +bool _mi_preloading(void); // true while the C runtime is not initialized yet +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap +void _mi_thread_done(mi_heap_t* heap); +void _mi_thread_data_collect(void); + +// os.c +void _mi_os_init(void); // called from process init +void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats); +void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats); +void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* stats); + +size_t _mi_os_page_size(void); +size_t _mi_os_good_alloc_size(size_t size); +bool _mi_os_has_overcommit(void); +bool _mi_os_has_virtual_reserve(void); + +bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); +bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats); +bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); +bool _mi_os_protect(void* addr, size_t size); +bool _mi_os_unprotect(void* addr, size_t size); +bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); +bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats); + +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats); +void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats); + +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); +bool _mi_os_use_large_page(size_t size, size_t alignment); +size_t _mi_os_large_page_size(void); + +void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); + +// arena.c +mi_arena_id_t _mi_arena_id_none(void); +void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid, mi_stats_t* stats); +void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); +bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id); +bool _mi_arena_contains(const void* p); +void _mi_arena_collect(bool force_purge, mi_stats_t* stats); +void _mi_arena_unsafe_destroy_all(mi_stats_t* stats); + +// "segment-map.c" +void _mi_segment_map_allocated_at(const mi_segment_t* segment); +void _mi_segment_map_freed_at(const mi_segment_t* segment); + +// "segment.c" +mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); +void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); +void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); +bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld); +void _mi_segment_thread_collect(mi_segments_tld_t* tld); + +#if MI_HUGE_PAGE_ABANDON +void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); +#else +void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); +#endif + +uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page +void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); +void _mi_abandoned_await_readers(void); +void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld); + +// "page.c" +void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; + +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks +void _mi_page_unfull(mi_page_t* page); +void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page +void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... +void _mi_heap_delayed_free_all(mi_heap_t* heap); +bool _mi_heap_delayed_free_partial(mi_heap_t* heap); +void _mi_heap_collect_retired(mi_heap_t* heap, bool force); + +void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); +bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); +size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); +void _mi_deferred_free(mi_heap_t* heap, bool force); + +void _mi_page_free_collect(mi_page_t* page,bool force); +void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments + +size_t _mi_bin_size(uint8_t bin); // for stats +uint8_t _mi_bin(size_t size); // for stats + +// "heap.c" +void _mi_heap_destroy_pages(mi_heap_t* heap); +void _mi_heap_collect_abandon(mi_heap_t* heap); +void _mi_heap_set_default_direct(mi_heap_t* heap); +bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid); +void _mi_heap_unsafe_destroy_all(void); + +// "stats.c" +void _mi_stats_done(mi_stats_t* stats); +mi_msecs_t _mi_clock_now(void); +mi_msecs_t _mi_clock_end(mi_msecs_t start); +mi_msecs_t _mi_clock_start(void); + +// "alloc.c" +void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_malloc_generic` +void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; +void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned` +void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept; +mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); +bool _mi_free_delayed_block(mi_block_t* block); +void _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration +void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); + +// option.c, c primitives +char _mi_toupper(char c); +int _mi_strnicmp(const char* s, const char* t, size_t n); +void _mi_strlcpy(char* dest, const char* src, size_t dest_size); +void _mi_strlcat(char* dest, const char* src, size_t dest_size); +size_t _mi_strlen(const char* s); +size_t _mi_strnlen(const char* s, size_t max_len); + + +#if MI_DEBUG>1 +bool _mi_page_is_valid(mi_page_t* page); +#endif + + +// ------------------------------------------------------ +// Branches +// ------------------------------------------------------ + +#if defined(__GNUC__) || defined(__clang__) +#define mi_unlikely(x) (__builtin_expect(!!(x),false)) +#define mi_likely(x) (__builtin_expect(!!(x),true)) +#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) +#define mi_unlikely(x) (x) [[unlikely]] +#define mi_likely(x) (x) [[likely]] +#else +#define mi_unlikely(x) (x) +#define mi_likely(x) (x) +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + + +/* ----------------------------------------------------------- + Error codes passed to `_mi_fatal_error` + All are recoverable but EFAULT is a serious error and aborts by default in secure mode. + For portability define undefined error codes using common Unix codes: + +----------------------------------------------------------- */ +#include +#ifndef EAGAIN // double free +#define EAGAIN (11) +#endif +#ifndef ENOMEM // out of memory +#define ENOMEM (12) +#endif +#ifndef EFAULT // corrupted free-list or meta-data +#define EFAULT (14) +#endif +#ifndef EINVAL // trying to free an invalid pointer +#define EINVAL (22) +#endif +#ifndef EOVERFLOW // count*size overflow +#define EOVERFLOW (75) +#endif + + +/* ----------------------------------------------------------- + Inlined definitions +----------------------------------------------------------- */ +#define MI_UNUSED(x) (void)(x) +#if (MI_DEBUG>0) +#define MI_UNUSED_RELEASE(x) +#else +#define MI_UNUSED_RELEASE(x) MI_UNUSED(x) +#endif + +#define MI_INIT4(x) x(),x(),x(),x() +#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x) +#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x) +#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x) +#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) +#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) +#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) + + +#include +// initialize a local variable to zero; use memset as compilers optimize constant sized memset's +#define _mi_memzero_var(x) memset(&x,0,sizeof(x)) + +// Is `x` a power of two? (0 is considered a power of two) +static inline bool _mi_is_power_of_two(uintptr_t x) { + return ((x & (x - 1)) == 0); +} + +// Is a pointer aligned? +static inline bool _mi_is_aligned(void* p, size_t alignment) { + mi_assert_internal(alignment != 0); + return (((uintptr_t)p % alignment) == 0); +} + +// Align upwards +static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return ((sz + mask) & ~mask); + } + else { + return (((sz + mask)/alignment)*alignment); + } +} + +// Align downwards +static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return (sz & ~mask); + } + else { + return ((sz / alignment) * alignment); + } +} + +// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`. +static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { + mi_assert_internal(divider != 0); + return (divider == 0 ? size : ((size + divider - 1) / divider)); +} + +// Is memory zero initialized? +static inline bool mi_mem_is_zero(const void* p, size_t size) { + for (size_t i = 0; i < size; i++) { + if (((uint8_t*)p)[i] != 0) return false; + } + return true; +} + + +// Align a byte size to a size in _machine words_, +// i.e. byte size == `wsize*sizeof(void*)`. +static inline size_t _mi_wsize_from_size(size_t size) { + mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t)); + return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); +} + +// Overflow detecting multiply +#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) +#include // UINT_MAX, ULONG_MAX +#if defined(_CLOCK_T) // for Illumos +#undef _CLOCK_T +#endif +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { + #if (SIZE_MAX == ULONG_MAX) + return __builtin_umull_overflow(count, size, (unsigned long *)total); + #elif (SIZE_MAX == UINT_MAX) + return __builtin_umul_overflow(count, size, (unsigned int *)total); + #else + return __builtin_umulll_overflow(count, size, (unsigned long long *)total); + #endif +} +#else /* __builtin_umul_overflow is unavailable */ +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { + #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) + *total = count * size; + // note: gcc/clang optimize this to directly check the overflow flag + return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); +} +#endif + +// Safe multiply `count*size` into `total`; return `true` on overflow. +static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) { + if (count==1) { // quick check for the case where count is one (common for C++ allocators) + *total = size; + return false; + } + else if mi_unlikely(mi_mul_overflow(count, size, total)) { + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu * %zu bytes)\n", count, size); + #endif + *total = SIZE_MAX; + return true; + } + else return false; +} + + +/*---------------------------------------------------------------------------------------- + Heap functions +------------------------------------------------------------------------------------------- */ + +extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap + +static inline bool mi_heap_is_backing(const mi_heap_t* heap) { + return (heap->tld->heap_backing == heap); +} + +static inline bool mi_heap_is_initialized(mi_heap_t* heap) { + mi_assert_internal(heap != NULL); + return (heap != &_mi_heap_empty); +} + +static inline uintptr_t _mi_ptr_cookie(const void* p) { + extern mi_heap_t _mi_heap_main; + mi_assert_internal(_mi_heap_main.cookie != 0); + return ((uintptr_t)p ^ _mi_heap_main.cookie); +} + +/* ----------------------------------------------------------- + Pages +----------------------------------------------------------- */ + +static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) { + mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + const size_t idx = _mi_wsize_from_size(size); + mi_assert_internal(idx < MI_PAGES_DIRECT); + return heap->pages_free_direct[idx]; +} + +// Segment that contains the pointer +// Large aligned blocks may be aligned at N*MI_SEGMENT_SIZE (inside a huge segment > MI_SEGMENT_SIZE), +// and we need align "down" to the segment info which is `MI_SEGMENT_SIZE` bytes before it; +// therefore we align one byte before `p`. +static inline mi_segment_t* _mi_ptr_segment(const void* p) { + mi_assert_internal(p != NULL); + return (mi_segment_t*)(((uintptr_t)p - 1) & ~MI_SEGMENT_MASK); +} + +static inline mi_page_t* mi_slice_to_page(mi_slice_t* s) { + mi_assert_internal(s->slice_offset== 0 && s->slice_count > 0); + return (mi_page_t*)(s); +} + +static inline mi_slice_t* mi_page_to_slice(mi_page_t* p) { + mi_assert_internal(p->slice_offset== 0 && p->slice_count > 0); + return (mi_slice_t*)(p); +} + +// Segment belonging to a page +static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { + mi_segment_t* segment = _mi_ptr_segment(page); + mi_assert_internal(segment == NULL || ((mi_slice_t*)page >= segment->slices && (mi_slice_t*)page < segment->slices + segment->slice_entries)); + return segment; +} + +static inline mi_slice_t* mi_slice_first(const mi_slice_t* slice) { + mi_slice_t* start = (mi_slice_t*)((uint8_t*)slice - slice->slice_offset); + mi_assert_internal(start >= _mi_ptr_segment(slice)->slices); + mi_assert_internal(start->slice_offset == 0); + mi_assert_internal(start + start->slice_count > slice); + return start; +} + +// Get the page containing the pointer (performance critical as it is called in mi_free) +static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { + mi_assert_internal(p > (void*)segment); + ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; + mi_assert_internal(diff > 0 && diff <= (ptrdiff_t)MI_SEGMENT_SIZE); + size_t idx = (size_t)diff >> MI_SEGMENT_SLICE_SHIFT; + mi_assert_internal(idx <= segment->slice_entries); + mi_slice_t* slice0 = (mi_slice_t*)&segment->slices[idx]; + mi_slice_t* slice = mi_slice_first(slice0); // adjust to the block that holds the page data + mi_assert_internal(slice->slice_offset == 0); + mi_assert_internal(slice >= segment->slices && slice < segment->slices + segment->slice_entries); + return mi_slice_to_page(slice); +} + +// Quick page start for initialized pages +static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) { + return _mi_segment_page_start(segment, page, page_size); +} + +// Get the page containing the pointer +static inline mi_page_t* _mi_ptr_page(void* p) { + return _mi_segment_page_of(_mi_ptr_segment(p), p); +} + +// Get the block size of a page (special case for huge objects) +static inline size_t mi_page_block_size(const mi_page_t* page) { + const size_t bsize = page->xblock_size; + mi_assert_internal(bsize > 0); + if mi_likely(bsize < MI_HUGE_BLOCK_SIZE) { + return bsize; + } + else { + size_t psize; + _mi_segment_page_start(_mi_page_segment(page), page, &psize); + return psize; + } +} + +static inline bool mi_page_is_huge(const mi_page_t* page) { + return (_mi_page_segment(page)->kind == MI_SEGMENT_HUGE); +} + +// Get the usable block size of a page without fixed padding. +// This may still include internal padding due to alignment and rounding up size classes. +static inline size_t mi_page_usable_block_size(const mi_page_t* page) { + return mi_page_block_size(page) - MI_PADDING_SIZE; +} + +// size of a segment +static inline size_t mi_segment_size(mi_segment_t* segment) { + return segment->segment_slices * MI_SEGMENT_SLICE_SIZE; +} + +static inline uint8_t* mi_segment_end(mi_segment_t* segment) { + return (uint8_t*)segment + mi_segment_size(segment); +} + +// Thread free access +static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { + return (mi_block_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & ~3); +} + +static inline mi_delayed_t mi_page_thread_free_flag(const mi_page_t* page) { + return (mi_delayed_t)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & 3); +} + +// Heap access +static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { + return (mi_heap_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xheap)); +} + +static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) { + mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING); + mi_atomic_store_release(&page->xheap,(uintptr_t)heap); +} + +// Thread free flag helpers +static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { + return (mi_block_t*)(tf & ~0x03); +} +static inline mi_delayed_t mi_tf_delayed(mi_thread_free_t tf) { + return (mi_delayed_t)(tf & 0x03); +} +static inline mi_thread_free_t mi_tf_make(mi_block_t* block, mi_delayed_t delayed) { + return (mi_thread_free_t)((uintptr_t)block | (uintptr_t)delayed); +} +static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) { + return mi_tf_make(mi_tf_block(tf),delayed); +} +static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t* block) { + return mi_tf_make(block, mi_tf_delayed(tf)); +} + +// are all blocks in a page freed? +// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. +static inline bool mi_page_all_free(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return (page->used == 0); +} + +// are there any available blocks? +static inline bool mi_page_has_any_available(const mi_page_t* page) { + mi_assert_internal(page != NULL && page->reserved > 0); + return (page->used < page->reserved || (mi_page_thread_free(page) != NULL)); +} + +// are there immediately available blocks, i.e. blocks available on the free list. +static inline bool mi_page_immediate_available(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return (page->free != NULL); +} + +// is more than 7/8th of a page in use? +static inline bool mi_page_mostly_used(const mi_page_t* page) { + if (page==NULL) return true; + uint16_t frac = page->reserved / 8U; + return (page->reserved - page->used <= frac); +} + +static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) { + return &((mi_heap_t*)heap)->pages[_mi_bin(size)]; +} + + + +//----------------------------------------------------------- +// Page flags +//----------------------------------------------------------- +static inline bool mi_page_is_in_full(const mi_page_t* page) { + return page->flags.x.in_full; +} + +static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { + page->flags.x.in_full = in_full; +} + +static inline bool mi_page_has_aligned(const mi_page_t* page) { + return page->flags.x.has_aligned; +} + +static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { + page->flags.x.has_aligned = has_aligned; +} + + +/* ------------------------------------------------------------------- +Encoding/Decoding the free list next pointers + +This is to protect against buffer overflow exploits where the +free list is mutated. Many hardened allocators xor the next pointer `p` +with a secret key `k1`, as `p^k1`. This prevents overwriting with known +values but might be still too weak: if the attacker can guess +the pointer `p` this can reveal `k1` (since `p^k1^p == k1`). +Moreover, if multiple blocks can be read as well, the attacker can +xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot +about the pointers (and subsequently `k1`). + +Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<> (MI_INTPTR_BITS - shift)))); +} +static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) { + shift %= MI_INTPTR_BITS; + return (shift==0 ? x : ((x >> shift) | (x << (MI_INTPTR_BITS - shift)))); +} + +static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { + void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]); + return (p==null ? NULL : p); +} + +static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { + uintptr_t x = (uintptr_t)(p==NULL ? null : p); + return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; +} + +static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { + mi_track_mem_defined(block,sizeof(mi_block_t)); + mi_block_t* next; + #ifdef MI_ENCODE_FREELIST + next = (mi_block_t*)mi_ptr_decode(null, block->next, keys); + #else + MI_UNUSED(keys); MI_UNUSED(null); + next = (mi_block_t*)block->next; + #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); + return next; +} + +static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { + mi_track_mem_undefined(block,sizeof(mi_block_t)); + #ifdef MI_ENCODE_FREELIST + block->next = mi_ptr_encode(null, next, keys); + #else + MI_UNUSED(keys); MI_UNUSED(null); + block->next = (mi_encoded_t)next; + #endif + mi_track_mem_noaccess(block,sizeof(mi_block_t)); +} + +static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { + #ifdef MI_ENCODE_FREELIST + mi_block_t* next = mi_block_nextx(page,block,page->keys); + // check for free list corruption: is `next` at least in the same page? + // TODO: check if `next` is `page->block_size` aligned? + if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) { + _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); + next = NULL; + } + return next; + #else + MI_UNUSED(page); + return mi_block_nextx(page,block,NULL); + #endif +} + +static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { + #ifdef MI_ENCODE_FREELIST + mi_block_set_nextx(page,block,next, page->keys); + #else + MI_UNUSED(page); + mi_block_set_nextx(page,block,next,NULL); + #endif +} + + +// ------------------------------------------------------------------- +// commit mask +// ------------------------------------------------------------------- + +static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + cm->mask[i] = 0; + } +} + +static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + cm->mask[i] = ~((size_t)0); + } +} + +static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + if (cm->mask[i] != 0) return false; + } + return true; +} + +static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) { + for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + if (cm->mask[i] != ~((size_t)0)) return false; + } + return true; +} + +// defined in `segment.c`: +size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total); +size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx); + +#define mi_commit_mask_foreach(cm,idx,count) \ + idx = 0; \ + while ((count = _mi_commit_mask_next_run(cm,&idx)) > 0) { + +#define mi_commit_mask_foreach_end() \ + idx += count; \ + } + + + +/* ----------------------------------------------------------- + memory id's +----------------------------------------------------------- */ + +static inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) { + mi_memid_t memid; + _mi_memzero_var(memid); + memid.memkind = memkind; + return memid; +} + +static inline mi_memid_t _mi_memid_none(void) { + return _mi_memid_create(MI_MEM_NONE); +} + +static inline mi_memid_t _mi_memid_create_os(bool committed, bool is_zero, bool is_large) { + mi_memid_t memid = _mi_memid_create(MI_MEM_OS); + memid.initially_committed = committed; + memid.initially_zero = is_zero; + memid.is_pinned = is_large; + return memid; +} + + +// ------------------------------------------------------------------- +// Fast "random" shuffle +// ------------------------------------------------------------------- + +static inline uintptr_t _mi_random_shuffle(uintptr_t x) { + if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros +#if (MI_INTPTR_SIZE==8) + // by Sebastiano Vigna, see: + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9UL; + x ^= x >> 27; + x *= 0x94d049bb133111ebUL; + x ^= x >> 31; +#elif (MI_INTPTR_SIZE==4) + // by Chris Wellons, see: + x ^= x >> 16; + x *= 0x7feb352dUL; + x ^= x >> 15; + x *= 0x846ca68bUL; + x ^= x >> 16; +#endif + return x; +} + +// ------------------------------------------------------------------- +// Optimize numa node access for the common case (= one node) +// ------------------------------------------------------------------- + +int _mi_os_numa_node_get(mi_os_tld_t* tld); +size_t _mi_os_numa_node_count_get(void); + +extern _Atomic(size_t) _mi_numa_node_count; +static inline int _mi_os_numa_node(mi_os_tld_t* tld) { + if mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1) { return 0; } + else return _mi_os_numa_node_get(tld); +} +static inline size_t _mi_os_numa_node_count(void) { + const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count); + if mi_likely(count > 0) { return count; } + else return _mi_os_numa_node_count_get(); +} + + + +// ----------------------------------------------------------------------- +// Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero) +// ----------------------------------------------------------------------- + +#if defined(__GNUC__) + +#include // LONG_MAX +#define MI_HAVE_FAST_BITSCAN +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (INTPTR_MAX == LONG_MAX) + return __builtin_clzl(x); +#else + return __builtin_clzll(x); +#endif +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (INTPTR_MAX == LONG_MAX) + return __builtin_ctzl(x); +#else + return __builtin_ctzll(x); +#endif +} + +#elif defined(_MSC_VER) + +#include // LONG_MAX +#include // BitScanReverse64 +#define MI_HAVE_FAST_BITSCAN +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; + unsigned long idx; +#if (INTPTR_MAX == LONG_MAX) + _BitScanReverse(&idx, x); +#else + _BitScanReverse64(&idx, x); +#endif + return ((MI_INTPTR_BITS - 1) - idx); +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; + unsigned long idx; +#if (INTPTR_MAX == LONG_MAX) + _BitScanForward(&idx, x); +#else + _BitScanForward64(&idx, x); +#endif + return idx; +} + +#else +static inline size_t mi_ctz32(uint32_t x) { + // de Bruijn multiplication, see + static const unsigned char debruijn[32] = { + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 + }; + if (x==0) return 32; + return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27]; +} +static inline size_t mi_clz32(uint32_t x) { + // de Bruijn multiplication, see + static const uint8_t debruijn[32] = { + 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1, + 23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0 + }; + if (x==0) return 32; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27]; +} + +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (MI_INTPTR_BITS <= 32) + return mi_clz32((uint32_t)x); +#else + size_t count = mi_clz32((uint32_t)(x >> 32)); + if (count < 32) return count; + return (32 + mi_clz32((uint32_t)x)); +#endif +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (MI_INTPTR_BITS <= 32) + return mi_ctz32((uint32_t)x); +#else + size_t count = mi_ctz32((uint32_t)x); + if (count < 32) return count; + return (32 + mi_ctz32((uint32_t)(x>>32))); +#endif +} + +#endif + +// "bit scan reverse": Return index of the highest bit (or MI_INTPTR_BITS if `x` is zero) +static inline size_t mi_bsr(uintptr_t x) { + return (x==0 ? MI_INTPTR_BITS : MI_INTPTR_BITS - 1 - mi_clz(x)); +} + + +// --------------------------------------------------------------------------------- +// Provide our own `_mi_memcpy` for potential performance optimizations. +// +// For now, only on Windows with msvc/clang-cl we optimize to `rep movsb` if +// we happen to run on x86/x64 cpu's that have "fast short rep movsb" (FSRM) support +// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. +// --------------------------------------------------------------------------------- + +#if !MI_TRACK_ENABLED && defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) +#include +extern bool _mi_cpu_has_fsrm; +static inline void _mi_memcpy(void* dst, const void* src, size_t n) { + if (_mi_cpu_has_fsrm) { + __movsb((unsigned char*)dst, (const unsigned char*)src, n); + } + else { + memcpy(dst, src, n); + } +} +static inline void _mi_memzero(void* dst, size_t n) { + if (_mi_cpu_has_fsrm) { + __stosb((unsigned char*)dst, 0, n); + } + else { + memset(dst, 0, n); + } +} +#else +static inline void _mi_memcpy(void* dst, const void* src, size_t n) { + memcpy(dst, src, n); +} +static inline void _mi_memzero(void* dst, size_t n) { + memset(dst, 0, n); +} +#endif + +// ------------------------------------------------------------------------------- +// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned +// This is used for example in `mi_realloc`. +// ------------------------------------------------------------------------------- + +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) +// On GCC/CLang we provide a hint that the pointers are word aligned. +static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); + void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); + _mi_memcpy(adst, asrc, n); +} + +static inline void _mi_memzero_aligned(void* dst, size_t n) { + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + _mi_memzero(adst, n); +} +#else +// Default fallback on `_mi_memcpy` +static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); + _mi_memcpy(dst, src, n); +} + +static inline void _mi_memzero_aligned(void* dst, size_t n) { + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + _mi_memzero(dst, n); +} +#endif + + +#endif diff --git a/Include/internal/mimalloc/mimalloc/prim.h b/Include/internal/mimalloc/mimalloc/prim.h new file mode 100644 index 00000000000000..4b9e4dc4194d77 --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/prim.h @@ -0,0 +1,323 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PRIM_H +#define MIMALLOC_PRIM_H + + +// -------------------------------------------------------------------------- +// This file specifies the primitive portability API. +// Each OS/host needs to implement these primitives, see `src/prim` +// for implementations on Window, macOS, WASI, and Linux/Unix. +// +// note: on all primitive functions, we always have result parameters != NUL, and: +// addr != NULL and page aligned +// size > 0 and page aligned +// return value is an error code an int where 0 is success. +// -------------------------------------------------------------------------- + +// OS memory configuration +typedef struct mi_os_mem_config_s { + size_t page_size; // 4KiB + size_t large_page_size; // 2MiB + size_t alloc_granularity; // smallest allocation size (on Windows 64KiB) + bool has_overcommit; // can we reserve more memory than can be actually committed? + bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc) + bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) +} mi_os_mem_config_t; + +// Initialize +void _mi_prim_mem_init( mi_os_mem_config_t* config ); + +// Free OS memory +int _mi_prim_free(void* addr, size_t size ); + +// Allocate OS memory. Return NULL on error. +// The `try_alignment` is just a hint and the returned pointer does not have to be aligned. +// If `commit` is false, the virtual memory range only needs to be reserved (with no access) +// which will later be committed explicitly using `_mi_prim_commit`. +// `is_zero` is set to true if the memory was zero initialized (as on most OS's) +// pre: !commit => !allow_large +// try_alignment >= _mi_os_page_size() and a power of 2 +int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr); + +// Commit memory. Returns error code or 0 on success. +// For example, on Linux this would make the memory PROT_READ|PROT_WRITE. +// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows) +int _mi_prim_commit(void* addr, size_t size, bool* is_zero); + +// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true +// if the memory would need to be re-committed. For example, on Windows this is always true, +// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit. +// pre: needs_recommit != NULL +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit); + +// Reset memory. The range keeps being accessible but the content might be reset. +// Returns error code or 0 on success. +int _mi_prim_reset(void* addr, size_t size); + +// Protect memory. Returns error code or 0 on success. +int _mi_prim_protect(void* addr, size_t size, bool protect); + +// Allocate huge (1GiB) pages possibly associated with a NUMA node. +// `is_zero` is set to true if the memory was zero initialized (as on most OS's) +// pre: size > 0 and a multiple of 1GiB. +// numa_node is either negative (don't care), or a numa node number. +int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr); + +// Return the current NUMA node +size_t _mi_prim_numa_node(void); + +// Return the number of logical NUMA nodes +size_t _mi_prim_numa_node_count(void); + +// Clock ticks +mi_msecs_t _mi_prim_clock_now(void); + +// Return process information (only for statistics) +typedef struct mi_process_info_s { + mi_msecs_t elapsed; + mi_msecs_t utime; + mi_msecs_t stime; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; +} mi_process_info_t; + +void _mi_prim_process_info(mi_process_info_t* pinfo); + +// Default stderr output. (only for warnings etc. with verbose enabled) +// msg != NULL && _mi_strlen(msg) > 0 +void _mi_prim_out_stderr( const char* msg ); + +// Get an environment variable. (only for options) +// name != NULL, result != NULL, result_size >= 64 +bool _mi_prim_getenv(const char* name, char* result, size_t result_size); + + +// Fill a buffer with strong randomness; return `false` on error or if +// there is no strong randomization available. +bool _mi_prim_random_buf(void* buf, size_t buf_len); + +// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination. +void _mi_prim_thread_init_auto_done(void); + +// Called on process exit and may take action to clean up resources associated with the thread auto done. +void _mi_prim_thread_done_auto_done(void); + +// Called when the default heap for a thread changes +void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); + + +//------------------------------------------------------------------- +// Thread id: `_mi_prim_thread_id()` +// +// Getting the thread id should be performant as it is called in the +// fast path of `_mi_free` and we specialize for various platforms as +// inlined definitions. Regular code should call `init.c:_mi_thread_id()`. +// We only require _mi_prim_thread_id() to return a unique id +// for each thread (unequal to zero). +//------------------------------------------------------------------- + +// defined in `init.c`; do not use these directly +extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from +extern bool _mi_process_is_initialized; // has mi_process_init been called? + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; + +#if defined(_WIN32) + +#define WIN32_LEAN_AND_MEAN +#include +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + // Windows: works on Intel and ARM in both 32- and 64-bit + return (uintptr_t)NtCurrentTeb(); +} + +// We use assembly for a fast thread id on the main platforms. The TLS layout depends on +// both the OS and libc implementation so we use specific tests for each main platform. +// If you test on another platform and it works please send a PR :-) +// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. +#elif defined(__GNUC__) && ( \ + (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ + || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ + || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ + ) + +static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { + void* res; + const size_t ofs = (slot*sizeof(void*)); + #if defined(__i386__) + __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS + #elif defined(__APPLE__) && defined(__x86_64__) + __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS + #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI + #elif defined(__x86_64__) + __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS + #elif defined(__arm__) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + res = tcb[slot]; + #elif defined(__aarch64__) + void** tcb; MI_UNUSED(ofs); + #if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + res = tcb[slot]; + #endif + return res; +} + +// setting a tls slot is only used on macOS for now +static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { + const size_t ofs = (slot*sizeof(void*)); + #if defined(__i386__) + __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS + #elif defined(__APPLE__) && defined(__x86_64__) + __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS + #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI + #elif defined(__x86_64__) + __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS + #elif defined(__arm__) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + tcb[slot] = value; + #elif defined(__aarch64__) + void** tcb; MI_UNUSED(ofs); + #if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + tcb[slot] = value; + #endif +} + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + #if defined(__BIONIC__) + // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id + // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 + return (uintptr_t)mi_prim_tls_slot(1); + #else + // in all our other targets, slot 0 is the thread id + // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h + // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 + return (uintptr_t)mi_prim_tls_slot(0); + #endif +} + +#else + +// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + return (uintptr_t)&_mi_heap_default; +} + +#endif + + + +/* ---------------------------------------------------------------------------------------- +The thread local default heap: `_mi_prim_get_default_heap()` +This is inlined here as it is on the fast path for allocation functions. + +On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a +__thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures +that the storage will always be available (allocated on the thread stacks). + +On some platforms though we cannot use that when overriding `malloc` since the underlying +TLS implementation (or the loader) will call itself `malloc` on a first access and recurse. +We try to circumvent this in an efficient way: +- macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the + loader itself calls `malloc` even before the modules are initialized. +- OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS). +- DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323) +------------------------------------------------------------------------------------------- */ + +static inline mi_heap_t* mi_prim_get_default_heap(void); + +#if defined(MI_MALLOC_OVERRIDE) +#if defined(__APPLE__) // macOS + #define MI_TLS_SLOT 89 // seems unused? + // #define MI_TLS_RECURSE_GUARD 1 + // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) + // see +#elif defined(__OpenBSD__) + // use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) + // see + #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) + // #elif defined(__DragonFly__) + // #warning "mimalloc is not working correctly on DragonFly yet." + // #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) +#elif defined(__ANDROID__) + // See issue #381 + #define MI_TLS_PTHREAD +#endif +#endif + + +#if defined(MI_TLS_SLOT) + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT); + if mi_unlikely(heap == NULL) { + #ifdef __GNUC__ + __asm(""); // prevent conditional load of the address of _mi_heap_empty + #endif + heap = (mi_heap_t*)&_mi_heap_empty; + } + return heap; +} + +#elif defined(MI_TLS_PTHREAD_SLOT_OFS) + +static inline mi_heap_t** mi_prim_tls_pthread_heap_slot(void) { + pthread_t self = pthread_self(); + #if defined(__DragonFly__) + if (self==NULL) return NULL; + #endif + return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS); +} + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t** pheap = mi_prim_tls_pthread_heap_slot(); + if mi_unlikely(pheap == NULL) return _mi_heap_main_get(); + mi_heap_t* heap = *pheap; + if mi_unlikely(heap == NULL) return (mi_heap_t*)&_mi_heap_empty; + return heap; +} + +#elif defined(MI_TLS_PTHREAD) + +extern pthread_key_t _mi_heap_default_key; +static inline mi_heap_t* mi_prim_get_default_heap(void) { + mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key)); + return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); +} + +#else // default using a thread local variable; used on most platforms. + +static inline mi_heap_t* mi_prim_get_default_heap(void) { + #if defined(MI_TLS_RECURSE_GUARD) + if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get(); + #endif + return _mi_heap_default; +} + +#endif // mi_prim_get_default_heap() + + + +#endif // MIMALLOC_PRIM_H diff --git a/Include/internal/mimalloc/mimalloc/track.h b/Include/internal/mimalloc/mimalloc/track.h new file mode 100644 index 00000000000000..fa1a048d846a9c --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/track.h @@ -0,0 +1,147 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_TRACK_H +#define MIMALLOC_TRACK_H + +/* ------------------------------------------------------------------------------------------------------ +Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers. +These can be defined for tracking allocation: + + #define mi_track_malloc_size(p,reqsize,size,zero) + #define mi_track_free_size(p,_size) + +The macros are set up such that the size passed to `mi_track_free_size` +always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`). +The `reqsize` is what the user requested, and `size >= reqsize`. +The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled, +or otherwise it is the usable block size which may be larger than the original request. +Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc). +The `zero` parameter is `true` if the allocated block is zero initialized. + +Optional: + + #define mi_track_align(p,alignedp,offset,size) + #define mi_track_resize(p,oldsize,newsize) + #define mi_track_init() + +The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block. +The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`). +The `mi_track_resize` is currently unused but could be called on reallocations within a block. +`mi_track_init` is called at program start. + +The following macros are for tools like asan and valgrind to track whether memory is +defined, undefined, or not accessible at all: + + #define mi_track_mem_defined(p,size) + #define mi_track_mem_undefined(p,size) + #define mi_track_mem_noaccess(p,size) + +-------------------------------------------------------------------------------------------------------*/ + +#if MI_TRACK_VALGRIND +// valgrind tool + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on heap_destroy +#define MI_TRACK_TOOL "valgrind" + +#include +#include + +#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero) +#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/) +#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/) +#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size) +#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) +#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) + +#elif MI_TRACK_ASAN +// address sanitizer + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "asan" + +#include + +#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size) +#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size) +#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size) + +#elif MI_TRACK_ETW +// windows event tracing + +#define MI_TRACK_ENABLED 1 +#define MI_TRACK_HEAP_DESTROY 1 +#define MI_TRACK_TOOL "ETW" + +#define WIN32_LEAN_AND_MEAN +#include +#include "../src/prim/windows/etw.h" + +#define mi_track_init() EventRegistermicrosoft_windows_mimalloc(); +#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size) +#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size) + +#else +// no tracking + +#define MI_TRACK_ENABLED 0 +#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_TOOL "none" + +#define mi_track_malloc_size(p,reqsize,size,zero) +#define mi_track_free_size(p,_size) + +#endif + +// ------------------- +// Utility definitions + +#ifndef mi_track_resize +#define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false) +#endif + +#ifndef mi_track_align +#define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset) +#endif + +#ifndef mi_track_init +#define mi_track_init() +#endif + +#ifndef mi_track_mem_defined +#define mi_track_mem_defined(p,size) +#endif + +#ifndef mi_track_mem_undefined +#define mi_track_mem_undefined(p,size) +#endif + +#ifndef mi_track_mem_noaccess +#define mi_track_mem_noaccess(p,size) +#endif + + +#if MI_PADDING +#define mi_track_malloc(p,reqsize,zero) \ + if ((p)!=NULL) { \ + mi_assert_internal(mi_usable_size(p)==(reqsize)); \ + mi_track_malloc_size(p,reqsize,reqsize,zero); \ + } +#else +#define mi_track_malloc(p,reqsize,zero) \ + if ((p)!=NULL) { \ + mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ + mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ + } +#endif + +#endif diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h new file mode 100644 index 00000000000000..7616f37e4b978f --- /dev/null +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -0,0 +1,670 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_TYPES_H +#define MIMALLOC_TYPES_H + +// -------------------------------------------------------------------------- +// This file contains the main type definitions for mimalloc: +// mi_heap_t : all data for a thread-local heap, contains +// lists of all managed heap pages. +// mi_segment_t : a larger chunk of memory (32GiB) from where pages +// are allocated. +// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from +// where objects are allocated. +// -------------------------------------------------------------------------- + + +#include // ptrdiff_t +#include // uintptr_t, uint16_t, etc +#include "mimalloc/atomic.h" // _Atomic + +#ifdef _MSC_VER +#pragma warning(disable:4214) // bitfield is not int +#endif + +// Minimal alignment necessary. On most platforms 16 bytes are needed +// due to SSE registers for example. This must be at least `sizeof(void*)` +#ifndef MI_MAX_ALIGN_SIZE +#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) +#endif + +// ------------------------------------------------------ +// Variants +// ------------------------------------------------------ + +// Define NDEBUG in the release version to disable assertions. +// #define NDEBUG + +// Define MI_TRACK_ to enable tracking support +// #define MI_TRACK_VALGRIND 1 +// #define MI_TRACK_ASAN 1 +// #define MI_TRACK_ETW 1 + +// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). +// #define MI_STAT 1 + +// Define MI_SECURE to enable security mitigations +// #define MI_SECURE 1 // guard page around metadata +// #define MI_SECURE 2 // guard page around each mimalloc page +// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) +// #define MI_SECURE 4 // checks for double free. (may be more expensive) + +#if !defined(MI_SECURE) +#define MI_SECURE 0 +#endif + +// Define MI_DEBUG for debug mode +// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. +// #define MI_DEBUG 2 // + internal assertion checks +// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON) +#if !defined(MI_DEBUG) +#if !defined(NDEBUG) || defined(_DEBUG) +#define MI_DEBUG 2 +#else +#define MI_DEBUG 0 +#endif +#endif + +// Reserve extra padding at the end of each block to be more resilient against heap block overflows. +// The padding can detect buffer overflow on free. +#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) +#define MI_PADDING 1 +#endif + +// Check padding bytes; allows byte-precise buffer overflow detection +#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1) +#define MI_PADDING_CHECK 1 +#endif + + +// Encoded free lists allow detection of corrupted free lists +// and can detect buffer overflows, modify after free, and double `free`s. +#if (MI_SECURE>=3 || MI_DEBUG>=1) +#define MI_ENCODE_FREELIST 1 +#endif + + +// We used to abandon huge pages but to eagerly deallocate if freed from another thread, +// but that makes it not possible to visit them during a heap walk or include them in a +// `mi_heap_destroy`. We therefore instead reset/decommit the huge blocks if freed from +// another thread so most memory is available until it gets properly freed by the owning thread. +// #define MI_HUGE_PAGE_ABANDON 1 + + +// ------------------------------------------------------ +// Platform specific values +// ------------------------------------------------------ + +// ------------------------------------------------------ +// Size of a pointer. +// We assume that `sizeof(void*)==sizeof(intptr_t)` +// and it holds for all platforms we know of. +// +// However, the C standard only requires that: +// p == (void*)((intptr_t)p)) +// but we also need: +// i == (intptr_t)((void*)i) +// or otherwise one might define an intptr_t type that is larger than a pointer... +// ------------------------------------------------------ + +#if INTPTR_MAX > INT64_MAX +# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example) +#elif INTPTR_MAX == INT64_MAX +# define MI_INTPTR_SHIFT (3) +#elif INTPTR_MAX == INT32_MAX +# define MI_INTPTR_SHIFT (2) +#else +#error platform pointers must be 32, 64, or 128 bits +#endif + +#if SIZE_MAX == UINT64_MAX +# define MI_SIZE_SHIFT (3) +typedef int64_t mi_ssize_t; +#elif SIZE_MAX == UINT32_MAX +# define MI_SIZE_SHIFT (2) +typedef int32_t mi_ssize_t; +#else +#error platform objects must be 32 or 64 bits +#endif + +#if (SIZE_MAX/2) > LONG_MAX +# define MI_ZU(x) x##ULL +# define MI_ZI(x) x##LL +#else +# define MI_ZU(x) x##UL +# define MI_ZI(x) x##L +#endif + +#define MI_INTPTR_SIZE (1< 4 +#define MI_SEGMENT_SHIFT ( 9 + MI_SEGMENT_SLICE_SHIFT) // 32MiB +#else +#define MI_SEGMENT_SHIFT ( 7 + MI_SEGMENT_SLICE_SHIFT) // 4MiB on 32-bit +#endif + +#define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64KiB +#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB + + +// Derived constants +#define MI_SEGMENT_SIZE (MI_ZU(1)<= 655360) +#error "mimalloc internal: define more bins" +#endif + +// Maximum slice offset (15) +#define MI_MAX_SLICE_OFFSET ((MI_ALIGNMENT_MAX / MI_SEGMENT_SLICE_SIZE) - 1) + +// Used as a special value to encode block sizes in 32 bits. +#define MI_HUGE_BLOCK_SIZE ((uint32_t)(2*MI_GiB)) + +// blocks up to this size are always allocated aligned +#define MI_MAX_ALIGN_GUARANTEE (8*MI_MAX_ALIGN_SIZE) + +// Alignments over MI_ALIGNMENT_MAX are allocated in dedicated huge page segments +#define MI_ALIGNMENT_MAX (MI_SEGMENT_SIZE >> 1) + + +// ------------------------------------------------------ +// Mimalloc pages contain allocated blocks +// ------------------------------------------------------ + +// The free lists use encoded next fields +// (Only actually encodes when MI_ENCODED_FREELIST is defined.) +typedef uintptr_t mi_encoded_t; + +// thread id's +typedef size_t mi_threadid_t; + +// free lists contain blocks +typedef struct mi_block_s { + mi_encoded_t next; +} mi_block_t; + + +// The delayed flags are used for efficient multi-threaded free-ing +typedef enum mi_delayed_e { + MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list + MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap + MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list + MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim +} mi_delayed_t; + + +// The `in_full` and `has_aligned` page flags are put in a union to efficiently +// test if both are false (`full_aligned == 0`) in the `mi_free` routine. +#if !MI_TSAN +typedef union mi_page_flags_s { + uint8_t full_aligned; + struct { + uint8_t in_full : 1; + uint8_t has_aligned : 1; + } x; +} mi_page_flags_t; +#else +// under thread sanitizer, use a byte for each flag to suppress warning, issue #130 +typedef union mi_page_flags_s { + uint16_t full_aligned; + struct { + uint8_t in_full; + uint8_t has_aligned; + } x; +} mi_page_flags_t; +#endif + +// Thread free list. +// We use the bottom 2 bits of the pointer for mi_delayed_t flags +typedef uintptr_t mi_thread_free_t; + +// A page contains blocks of one specific size (`block_size`). +// Each page has three list of free blocks: +// `free` for blocks that can be allocated, +// `local_free` for freed blocks that are not yet available to `mi_malloc` +// `thread_free` for freed blocks by other threads +// The `local_free` and `thread_free` lists are migrated to the `free` list +// when it is exhausted. The separate `local_free` list is necessary to +// implement a monotonic heartbeat. The `thread_free` list is needed for +// avoiding atomic operations in the common case. +// +// +// `used - |thread_free|` == actual blocks that are in use (alive) +// `used - |thread_free| + |free| + |local_free| == capacity` +// +// We don't count `freed` (as |free|) but use `used` to reduce +// the number of memory accesses in the `mi_page_all_free` function(s). +// +// Notes: +// - Access is optimized for `mi_free` and `mi_page_alloc` (in `alloc.c`) +// - Using `uint16_t` does not seem to slow things down +// - The size is 8 words on 64-bit which helps the page index calculations +// (and 10 words on 32-bit, and encoded free lists add 2 words. Sizes 10 +// and 12 are still good for address calculation) +// - To limit the structure size, the `xblock_size` is 32-bits only; for +// blocks > MI_HUGE_BLOCK_SIZE the size is determined from the segment page size +// - `thread_free` uses the bottom bits as a delayed-free flags to optimize +// concurrent frees where only the first concurrent free adds to the owning +// heap `thread_delayed_free` list (see `alloc.c:mi_free_block_mt`). +// The invariant is that no-delayed-free is only set if there is +// at least one block that will be added, or as already been added, to +// the owning heap `thread_delayed_free` list. This guarantees that pages +// will be freed correctly even if only other threads free blocks. +typedef struct mi_page_s { + // "owned" by the segment + uint32_t slice_count; // slices in this page (0 if not a page) + uint32_t slice_offset; // distance from the actual page data slice (0 if a page) + uint8_t is_committed : 1; // `true` if the page virtual memory is committed + uint8_t is_zero_init : 1; // `true` if the page was initially zero initialized + + // layout like this to optimize access in `mi_malloc` and `mi_free` + uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` + uint16_t reserved; // number of blocks reserved in memory + mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits) + uint8_t free_is_zero : 1; // `true` if the blocks in the free list are zero initialized + uint8_t retire_expire : 7; // expiration count for retired blocks + + mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) + uint32_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) + uint32_t xblock_size; // size available in each block (always `>0`) + mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + #if (MI_ENCODE_FREELIST || MI_PADDING) + uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) or padding canary + #endif + + _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads + _Atomic(uintptr_t) xheap; + + struct mi_page_s* next; // next page owned by this thread with the same `block_size` + struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` + + // 64-bit 9 words, 32-bit 12 words, (+2 for secure) + #if MI_INTPTR_SIZE==8 + uintptr_t padding[1]; + #endif +} mi_page_t; + + + +// ------------------------------------------------------ +// Mimalloc segments contain mimalloc pages +// ------------------------------------------------------ + +typedef enum mi_page_kind_e { + MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment + MI_PAGE_MEDIUM, // medium blocks go into medium pages inside a segment + MI_PAGE_LARGE, // larger blocks go into a page of just one block + MI_PAGE_HUGE, // huge blocks (> 16 MiB) are put into a single page in a single segment. +} mi_page_kind_t; + +typedef enum mi_segment_kind_e { + MI_SEGMENT_NORMAL, // MI_SEGMENT_SIZE size with pages inside. + MI_SEGMENT_HUGE, // > MI_LARGE_SIZE_MAX segment with just one huge page inside. +} mi_segment_kind_t; + +// ------------------------------------------------------ +// A segment holds a commit mask where a bit is set if +// the corresponding MI_COMMIT_SIZE area is committed. +// The MI_COMMIT_SIZE must be a multiple of the slice +// size. If it is equal we have the most fine grained +// decommit (but setting it higher can be more efficient). +// The MI_MINIMAL_COMMIT_SIZE is the minimal amount that will +// be committed in one go which can be set higher than +// MI_COMMIT_SIZE for efficiency (while the decommit mask +// is still tracked in fine-grained MI_COMMIT_SIZE chunks) +// ------------------------------------------------------ + +#define MI_MINIMAL_COMMIT_SIZE (1*MI_SEGMENT_SLICE_SIZE) +#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) // 64KiB +#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) +#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS +#define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) + +#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS)) +#error "the segment size must be exactly divisible by the (commit size * size_t bits)" +#endif + +typedef struct mi_commit_mask_s { + size_t mask[MI_COMMIT_MASK_FIELD_COUNT]; +} mi_commit_mask_t; + +typedef mi_page_t mi_slice_t; +typedef int64_t mi_msecs_t; + + +// Memory can reside in arena's, direct OS allocated, or statically allocated. The memid keeps track of this. +typedef enum mi_memkind_e { + MI_MEM_NONE, // not allocated + MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example) + MI_MEM_STATIC, // allocated in a static area and should not be freed (for arena meta data for example) + MI_MEM_OS, // allocated from the OS + MI_MEM_OS_HUGE, // allocated as huge os pages + MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) + MI_MEM_ARENA // allocated from an arena (the usual case) +} mi_memkind_t; + +static inline bool mi_memkind_is_os(mi_memkind_t memkind) { + return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP); +} + +typedef struct mi_memid_os_info { + void* base; // actual base address of the block (used for offset aligned allocations) + size_t alignment; // alignment at allocation +} mi_memid_os_info_t; + +typedef struct mi_memid_arena_info { + size_t block_index; // index in the arena + mi_arena_id_t id; // arena id (>= 1) + bool is_exclusive; // the arena can only be used for specific arena allocations +} mi_memid_arena_info_t; + +typedef struct mi_memid_s { + union { + mi_memid_os_info_t os; // only used for MI_MEM_OS + mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA + } mem; + bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large OS pages) + bool initially_committed;// `true` if the memory was originally allocated as committed + bool initially_zero; // `true` if the memory was originally zero initialized + mi_memkind_t memkind; +} mi_memid_t; + + +// Segments are large allocated memory blocks (8mb on 64 bit) from +// the OS. Inside segments we allocated fixed size _pages_ that +// contain blocks. +typedef struct mi_segment_s { + // constant fields + mi_memid_t memid; // memory id for arena allocation + bool allow_decommit; + bool allow_purge; + size_t segment_size; + + // segment fields + mi_msecs_t purge_expire; + mi_commit_mask_t purge_mask; + mi_commit_mask_t commit_mask; + + _Atomic(struct mi_segment_s*) abandoned_next; + + // from here is zero initialized + struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`) + + size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) + size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long) + size_t used; // count of pages in use + uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` + + size_t segment_slices; // for huge segments this may be different from `MI_SLICES_PER_SEGMENT` + size_t segment_info_slices; // initial slices we are using segment info and possible guard pages. + + // layout like this to optimize access in `mi_free` + mi_segment_kind_t kind; + size_t slice_entries; // entries in the `slices` array, at most `MI_SLICES_PER_SEGMENT` + _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment + + mi_slice_t slices[MI_SLICES_PER_SEGMENT+1]; // one more for huge blocks with large alignment +} mi_segment_t; + + +// ------------------------------------------------------ +// Heaps +// Provide first-class heaps to allocate from. +// A heap just owns a set of pages for allocation and +// can only be allocate/reallocate from the thread that created it. +// Freeing blocks can be done from any thread though. +// Per thread, the segments are shared among its heaps. +// Per thread, there is always a default heap that is +// used for allocation; it is initialized to statically +// point to an empty heap to avoid initialization checks +// in the fast path. +// ------------------------------------------------------ + +// Thread local data +typedef struct mi_tld_s mi_tld_t; + +// Pages of a certain block size are held in a queue. +typedef struct mi_page_queue_s { + mi_page_t* first; + mi_page_t* last; + size_t block_size; +} mi_page_queue_t; + +#define MI_BIN_FULL (MI_BIN_HUGE+1) + +// Random context +typedef struct mi_random_cxt_s { + uint32_t input[16]; + uint32_t output[16]; + int output_available; + bool weak; +} mi_random_ctx_t; + + +// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows +#if (MI_PADDING) +typedef struct mi_padding_s { + uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) + uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) +} mi_padding_t; +#define MI_PADDING_SIZE (sizeof(mi_padding_t)) +#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) +#else +#define MI_PADDING_SIZE 0 +#define MI_PADDING_WSIZE 0 +#endif + +#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1) + + +// A heap owns a set of pages. +struct mi_heap_s { + mi_tld_t* tld; + mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") + _Atomic(mi_block_t*) thread_delayed_free; + mi_threadid_t thread_id; // thread this heap belongs too + mi_arena_id_t arena_id; // arena id if the heap belongs to a specific arena (or 0) + uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) + uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list + mi_random_ctx_t random; // random number context used for secure allocation + size_t page_count; // total number of pages in the `pages` queues. + size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) + size_t page_retired_max; // largest retired index into the `pages` array. + mi_heap_t* next; // list of heaps per thread + bool no_reclaim; // `true` if this heap should not reclaim abandoned pages +}; + + + +// ------------------------------------------------------ +// Debug +// ------------------------------------------------------ + +#if !defined(MI_DEBUG_UNINIT) +#define MI_DEBUG_UNINIT (0xD0) +#endif +#if !defined(MI_DEBUG_FREED) +#define MI_DEBUG_FREED (0xDF) +#endif +#if !defined(MI_DEBUG_PADDING) +#define MI_DEBUG_PADDING (0xDE) +#endif + +#if (MI_DEBUG) +// use our own assertion to print without memory allocation +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); +#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) +#else +#define mi_assert(x) +#endif + +#if (MI_DEBUG>1) +#define mi_assert_internal mi_assert +#else +#define mi_assert_internal(x) +#endif + +#if (MI_DEBUG>2) +#define mi_assert_expensive mi_assert +#else +#define mi_assert_expensive(x) +#endif + +// ------------------------------------------------------ +// Statistics +// ------------------------------------------------------ + +#ifndef MI_STAT +#if (MI_DEBUG>0) +#define MI_STAT 2 +#else +#define MI_STAT 0 +#endif +#endif + +typedef struct mi_stat_count_s { + int64_t allocated; + int64_t freed; + int64_t peak; + int64_t current; +} mi_stat_count_t; + +typedef struct mi_stat_counter_s { + int64_t total; + int64_t count; +} mi_stat_counter_t; + +typedef struct mi_stats_s { + mi_stat_count_t segments; + mi_stat_count_t pages; + mi_stat_count_t reserved; + mi_stat_count_t committed; + mi_stat_count_t reset; + mi_stat_count_t purged; + mi_stat_count_t page_committed; + mi_stat_count_t segments_abandoned; + mi_stat_count_t pages_abandoned; + mi_stat_count_t threads; + mi_stat_count_t normal; + mi_stat_count_t huge; + mi_stat_count_t large; + mi_stat_count_t malloc; + mi_stat_count_t segments_cache; + mi_stat_counter_t pages_extended; + mi_stat_counter_t mmap_calls; + mi_stat_counter_t commit_calls; + mi_stat_counter_t reset_calls; + mi_stat_counter_t purge_calls; + mi_stat_counter_t page_no_retire; + mi_stat_counter_t searches; + mi_stat_counter_t normal_count; + mi_stat_counter_t huge_count; + mi_stat_counter_t large_count; +#if MI_STAT>1 + mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; +#endif +} mi_stats_t; + + +void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); +void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); +void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); + +#if (MI_STAT) +#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) +#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) +#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) +#else +#define mi_stat_increase(stat,amount) (void)0 +#define mi_stat_decrease(stat,amount) (void)0 +#define mi_stat_counter_increase(stat,amount) (void)0 +#endif + +#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) +#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) +#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) + +// ------------------------------------------------------ +// Thread Local data +// ------------------------------------------------------ + +// A "span" is is an available range of slices. The span queues keep +// track of slice spans of at most the given `slice_count` (but more than the previous size class). +typedef struct mi_span_queue_s { + mi_slice_t* first; + mi_slice_t* last; + size_t slice_count; +} mi_span_queue_t; + +#define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SLICES_PER_SEGMENT) + +// OS thread local data +typedef struct mi_os_tld_s { + size_t region_idx; // start point for next allocation + mi_stats_t* stats; // points to tld stats +} mi_os_tld_t; + + +// Segments thread local data +typedef struct mi_segments_tld_s { + mi_span_queue_t spans[MI_SEGMENT_BIN_MAX+1]; // free slice spans inside segments + size_t count; // current number of segments; + size_t peak_count; // peak number of segments + size_t current_size; // current size of all segments + size_t peak_size; // peak size of all segments + mi_stats_t* stats; // points to tld stats + mi_os_tld_t* os; // points to os stats +} mi_segments_tld_t; + +// Thread local data +struct mi_tld_s { + unsigned long long heartbeat; // monotonic heartbeat count + bool recurse; // true if deferred was called; used to prevent infinite recursion. + mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted) + mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates) + mi_segments_tld_t segments; // segment tld + mi_os_tld_t os; // os tld + mi_stats_t stats; // statistics +}; + +#endif diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index b1afb2dc7be65e..3cc0afac4bd5b4 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -19,6 +19,42 @@ _PyIndex_Check(PyObject *obj) PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); +extern int _PyObject_HasLen(PyObject *o); + +/* === Sequence protocol ================================================ */ + +#define PY_ITERSEARCH_COUNT 1 +#define PY_ITERSEARCH_INDEX 2 +#define PY_ITERSEARCH_CONTAINS 3 + +/* Iterate over seq. + + Result depends on the operation: + + PY_ITERSEARCH_COUNT: return # of times obj appears in seq; -1 if + error. + PY_ITERSEARCH_INDEX: return 0-based index of first occurrence of + obj in seq; set ValueError and return -1 if none found; + also return -1 on error. + PY_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on + error. */ +extern Py_ssize_t _PySequence_IterSearch(PyObject *seq, + PyObject *obj, int operation); + +/* === Mapping protocol ================================================= */ + +extern int _PyObject_RealIsInstance(PyObject *inst, PyObject *cls); + +extern int _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls); + +// Convert Python int to Py_ssize_t. Do nothing if the argument is None. +// Export for '_bisect' shared extension. +PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); + +// Same as PyNumber_Index() but can return an instance of a subclass of int. +// Export for 'math' shared extension. +PyAPI_FUNC(PyObject*) _PyNumber_Index(PyObject *o); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 36277efe9c5ca5..f222d485e0b54b 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -10,7 +10,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_asdl.h" +#include "pycore_asdl.h" // _ASDL_SEQ_HEAD typedef struct _mod *mod_ty; @@ -51,6 +51,8 @@ typedef struct _pattern *pattern_ty; typedef struct _type_ignore *type_ignore_ty; +typedef struct _type_param *type_param_ty; + typedef struct { _ASDL_SEQ_HEAD @@ -147,6 +149,14 @@ typedef struct { asdl_type_ignore_seq *_Py_asdl_type_ignore_seq_new(Py_ssize_t size, PyArena *arena); +typedef struct { + _ASDL_SEQ_HEAD + type_param_ty typed_elements[1]; +} asdl_type_param_seq; + +asdl_type_param_seq *_Py_asdl_type_param_seq_new(Py_ssize_t size, PyArena + *arena); + enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, FunctionType_kind=4}; @@ -176,12 +186,13 @@ struct _mod { enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3, Return_kind=4, Delete_kind=5, Assign_kind=6, - AugAssign_kind=7, AnnAssign_kind=8, For_kind=9, - AsyncFor_kind=10, While_kind=11, If_kind=12, With_kind=13, - AsyncWith_kind=14, Match_kind=15, Raise_kind=16, Try_kind=17, - TryStar_kind=18, Assert_kind=19, Import_kind=20, - ImportFrom_kind=21, Global_kind=22, Nonlocal_kind=23, - Expr_kind=24, Pass_kind=25, Break_kind=26, Continue_kind=27}; + TypeAlias_kind=7, AugAssign_kind=8, AnnAssign_kind=9, + For_kind=10, AsyncFor_kind=11, While_kind=12, If_kind=13, + With_kind=14, AsyncWith_kind=15, Match_kind=16, + Raise_kind=17, Try_kind=18, TryStar_kind=19, Assert_kind=20, + Import_kind=21, ImportFrom_kind=22, Global_kind=23, + Nonlocal_kind=24, Expr_kind=25, Pass_kind=26, Break_kind=27, + Continue_kind=28}; struct _stmt { enum _stmt_kind kind; union { @@ -192,6 +203,7 @@ struct _stmt { asdl_expr_seq *decorator_list; expr_ty returns; string type_comment; + asdl_type_param_seq *type_params; } FunctionDef; struct { @@ -201,6 +213,7 @@ struct _stmt { asdl_expr_seq *decorator_list; expr_ty returns; string type_comment; + asdl_type_param_seq *type_params; } AsyncFunctionDef; struct { @@ -209,6 +222,7 @@ struct _stmt { asdl_keyword_seq *keywords; asdl_stmt_seq *body; asdl_expr_seq *decorator_list; + asdl_type_param_seq *type_params; } ClassDef; struct { @@ -225,6 +239,12 @@ struct _stmt { string type_comment; } Assign; + struct { + expr_ty name; + asdl_type_param_seq *type_params; + expr_ty value; + } TypeAlias; + struct { expr_ty target; operator_ty op; @@ -630,6 +650,30 @@ struct _type_ignore { } v; }; +enum _type_param_kind {TypeVar_kind=1, ParamSpec_kind=2, TypeVarTuple_kind=3}; +struct _type_param { + enum _type_param_kind kind; + union { + struct { + identifier name; + expr_ty bound; + } TypeVar; + + struct { + identifier name; + } ParamSpec; + + struct { + identifier name; + } TypeVarTuple; + + } v; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + // Note: these macros affect function definitions, not only call sites. mod_ty _PyAST_Module(asdl_stmt_seq * body, asdl_type_ignore_seq * type_ignores, @@ -640,19 +684,20 @@ mod_ty _PyAST_FunctionType(asdl_expr_seq * argtypes, expr_ty returns, PyArena *arena); stmt_ty _PyAST_FunctionDef(identifier name, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty - returns, string type_comment, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); + returns, string type_comment, asdl_type_param_seq * + type_params, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_AsyncFunctionDef(identifier name, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string - type_comment, int lineno, int col_offset, int + type_comment, asdl_type_param_seq * + type_params, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_ClassDef(identifier name, asdl_expr_seq * bases, asdl_keyword_seq * keywords, asdl_stmt_seq * body, - asdl_expr_seq * decorator_list, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); + asdl_expr_seq * decorator_list, asdl_type_param_seq * + type_params, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Return(expr_ty value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Delete(asdl_expr_seq * targets, int lineno, int col_offset, int @@ -660,6 +705,9 @@ stmt_ty _PyAST_Delete(asdl_expr_seq * targets, int lineno, int col_offset, int stmt_ty _PyAST_Assign(asdl_expr_seq * targets, expr_ty value, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_TypeAlias(expr_ty name, asdl_type_param_seq * type_params, + expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); @@ -844,6 +892,14 @@ pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena + *arena); PyObject* PyAST_mod2obj(mod_ty t); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index f15b4905eed14b..6ffd30aca7b11b 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -2,6 +2,9 @@ #ifndef Py_INTERNAL_AST_STATE_H #define Py_INTERNAL_AST_STATE_H + +#include "pycore_lock.h" // _PyOnceFlag + #ifdef __cplusplus extern "C" { #endif @@ -11,7 +14,8 @@ extern "C" { #endif struct ast_state { - int initialized; + _PyOnceFlag once; + int finalized; int recursion_depth; int recursion_limit; PyObject *AST_type; @@ -118,6 +122,7 @@ struct ast_state { PyObject *Not_type; PyObject *Or_singleton; PyObject *Or_type; + PyObject *ParamSpec_type; PyObject *Pass_type; PyObject *Pow_singleton; PyObject *Pow_type; @@ -137,7 +142,10 @@ struct ast_state { PyObject *TryStar_type; PyObject *Try_type; PyObject *Tuple_type; + PyObject *TypeAlias_type; PyObject *TypeIgnore_type; + PyObject *TypeVarTuple_type; + PyObject *TypeVar_type; PyObject *UAdd_singleton; PyObject *UAdd_type; PyObject *USub_singleton; @@ -166,6 +174,7 @@ struct ast_state { PyObject *bases; PyObject *body; PyObject *boolop_type; + PyObject *bound; PyObject *cases; PyObject *cause; PyObject *cls; @@ -243,6 +252,8 @@ struct ast_state { PyObject *type_comment; PyObject *type_ignore_type; PyObject *type_ignores; + PyObject *type_param_type; + PyObject *type_params; PyObject *unaryop_type; PyObject *upper; PyObject *value; diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h index b4663b396852f3..4dcda8f517c787 100644 --- a/Include/internal/pycore_atexit.h +++ b/Include/internal/pycore_atexit.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_ATEXIT_H #define Py_INTERNAL_ATEXIT_H + +#include "pycore_lock.h" // PyMutex + #ifdef __cplusplus extern "C" { #endif @@ -15,6 +18,7 @@ extern "C" { typedef void (*atexit_callbackfunc)(void); struct _atexit_runtime_state { + PyMutex mutex; #define NEXITFUNCS 32 atexit_callbackfunc callbacks[NEXITFUNCS]; int ncallbacks; @@ -24,7 +28,8 @@ struct _atexit_runtime_state { //################### // interpreter atexit -struct atexit_callback; +typedef void (*atexit_datacallbackfunc)(void *); + typedef struct atexit_callback { atexit_datacallbackfunc func; void *data; @@ -49,6 +54,11 @@ struct atexit_state { int callback_len; }; +// Export for '_xxinterpchannels' shared extension +PyAPI_FUNC(int) _Py_AtExit( + PyInterpreterState *interp, + atexit_datacallbackfunc func, + void *data); #ifdef __cplusplus } diff --git a/Include/internal/pycore_atomic.h b/Include/internal/pycore_atomic.h deleted file mode 100644 index 425d69f868b52b..00000000000000 --- a/Include/internal/pycore_atomic.h +++ /dev/null @@ -1,557 +0,0 @@ -#ifndef Py_ATOMIC_H -#define Py_ATOMIC_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "dynamic_annotations.h" /* _Py_ANNOTATE_MEMORY_ORDER */ -#include "pyconfig.h" - -#ifdef HAVE_STD_ATOMIC -# include -#endif - - -#if defined(_MSC_VER) -#include -#if defined(_M_IX86) || defined(_M_X64) -# include -#endif -#endif - -/* This is modeled after the atomics interface from C1x, according to - * the draft at - * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf. - * Operations and types are named the same except with a _Py_ prefix - * and have the same semantics. - * - * Beware, the implementations here are deep magic. - */ - -#if defined(HAVE_STD_ATOMIC) - -typedef enum _Py_memory_order { - _Py_memory_order_relaxed = memory_order_relaxed, - _Py_memory_order_acquire = memory_order_acquire, - _Py_memory_order_release = memory_order_release, - _Py_memory_order_acq_rel = memory_order_acq_rel, - _Py_memory_order_seq_cst = memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - atomic_uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - atomic_int _value; -} _Py_atomic_int; - -#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ - atomic_signal_fence(ORDER) - -#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ - atomic_thread_fence(ORDER) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - atomic_store_explicit(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER) - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - atomic_load_explicit(&((ATOMIC_VAL)->_value), ORDER) - -// Use builtin atomic operations in GCC >= 4.7 and clang -#elif defined(HAVE_BUILTIN_ATOMIC) - -typedef enum _Py_memory_order { - _Py_memory_order_relaxed = __ATOMIC_RELAXED, - _Py_memory_order_acquire = __ATOMIC_ACQUIRE, - _Py_memory_order_release = __ATOMIC_RELEASE, - _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL, - _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST -} _Py_memory_order; - -typedef struct _Py_atomic_address { - uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - int _value; -} _Py_atomic_int; - -#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ - __atomic_signal_fence(ORDER) - -#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ - __atomic_thread_fence(ORDER) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - (assert((ORDER) == __ATOMIC_RELAXED \ - || (ORDER) == __ATOMIC_SEQ_CST \ - || (ORDER) == __ATOMIC_RELEASE), \ - __atomic_store_n(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER)) - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - (assert((ORDER) == __ATOMIC_RELAXED \ - || (ORDER) == __ATOMIC_SEQ_CST \ - || (ORDER) == __ATOMIC_ACQUIRE \ - || (ORDER) == __ATOMIC_CONSUME), \ - __atomic_load_n(&((ATOMIC_VAL)->_value), ORDER)) - -/* Only support GCC (for expression statements) and x86 (for simple - * atomic semantics) and MSVC x86/x64/ARM */ -#elif defined(__GNUC__) && (defined(__i386__) || defined(__amd64)) -typedef enum _Py_memory_order { - _Py_memory_order_relaxed, - _Py_memory_order_acquire, - _Py_memory_order_release, - _Py_memory_order_acq_rel, - _Py_memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - int _value; -} _Py_atomic_int; - - -static __inline__ void -_Py_atomic_signal_fence(_Py_memory_order order) -{ - if (order != _Py_memory_order_relaxed) - __asm__ volatile("":::"memory"); -} - -static __inline__ void -_Py_atomic_thread_fence(_Py_memory_order order) -{ - if (order != _Py_memory_order_relaxed) - __asm__ volatile("mfence":::"memory"); -} - -/* Tell the race checker about this operation's effects. */ -static __inline__ void -_Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order) -{ - (void)address; /* shut up -Wunused-parameter */ - switch(order) { - case _Py_memory_order_release: - case _Py_memory_order_acq_rel: - case _Py_memory_order_seq_cst: - _Py_ANNOTATE_HAPPENS_BEFORE(address); - break; - case _Py_memory_order_relaxed: - case _Py_memory_order_acquire: - break; - } - switch(order) { - case _Py_memory_order_acquire: - case _Py_memory_order_acq_rel: - case _Py_memory_order_seq_cst: - _Py_ANNOTATE_HAPPENS_AFTER(address); - break; - case _Py_memory_order_relaxed: - case _Py_memory_order_release: - break; - } -} - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - __extension__ ({ \ - __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ - __typeof__(atomic_val->_value) new_val = NEW_VAL;\ - volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \ - _Py_memory_order order = ORDER; \ - _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ - \ - /* Perform the operation. */ \ - _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \ - switch(order) { \ - case _Py_memory_order_release: \ - _Py_atomic_signal_fence(_Py_memory_order_release); \ - /* fallthrough */ \ - case _Py_memory_order_relaxed: \ - *volatile_data = new_val; \ - break; \ - \ - case _Py_memory_order_acquire: \ - case _Py_memory_order_acq_rel: \ - case _Py_memory_order_seq_cst: \ - __asm__ volatile("xchg %0, %1" \ - : "+r"(new_val) \ - : "m"(atomic_val->_value) \ - : "memory"); \ - break; \ - } \ - _Py_ANNOTATE_IGNORE_WRITES_END(); \ - }) - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - __extension__ ({ \ - __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ - __typeof__(atomic_val->_value) result; \ - volatile __typeof__(result) *volatile_data = &atomic_val->_value; \ - _Py_memory_order order = ORDER; \ - _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ - \ - /* Perform the operation. */ \ - _Py_ANNOTATE_IGNORE_READS_BEGIN(); \ - switch(order) { \ - case _Py_memory_order_release: \ - case _Py_memory_order_acq_rel: \ - case _Py_memory_order_seq_cst: \ - /* Loads on x86 are not releases by default, so need a */ \ - /* thread fence. */ \ - _Py_atomic_thread_fence(_Py_memory_order_release); \ - break; \ - default: \ - /* No fence */ \ - break; \ - } \ - result = *volatile_data; \ - switch(order) { \ - case _Py_memory_order_acquire: \ - case _Py_memory_order_acq_rel: \ - case _Py_memory_order_seq_cst: \ - /* Loads on x86 are automatically acquire operations so */ \ - /* can get by with just a compiler fence. */ \ - _Py_atomic_signal_fence(_Py_memory_order_acquire); \ - break; \ - default: \ - /* No fence */ \ - break; \ - } \ - _Py_ANNOTATE_IGNORE_READS_END(); \ - result; \ - }) - -#elif defined(_MSC_VER) -/* _Interlocked* functions provide a full memory barrier and are therefore - enough for acq_rel and seq_cst. If the HLE variants aren't available - in hardware they will fall back to a full memory barrier as well. - - This might affect performance but likely only in some very specific and - hard to measure scenario. -*/ -#if defined(_M_IX86) || defined(_M_X64) -typedef enum _Py_memory_order { - _Py_memory_order_relaxed, - _Py_memory_order_acquire, - _Py_memory_order_release, - _Py_memory_order_acq_rel, - _Py_memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - volatile uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - volatile int _value; -} _Py_atomic_int; - - -#if defined(_M_X64) -#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \ - switch (ORDER) { \ - case _Py_memory_order_acquire: \ - _InterlockedExchange64_HLEAcquire((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ - break; \ - case _Py_memory_order_release: \ - _InterlockedExchange64_HLERelease((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ - break; \ - default: \ - _InterlockedExchange64((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ - break; \ - } -#else -#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) ((void)0); -#endif - -#define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \ - switch (ORDER) { \ - case _Py_memory_order_acquire: \ - _InterlockedExchange_HLEAcquire((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ - break; \ - case _Py_memory_order_release: \ - _InterlockedExchange_HLERelease((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ - break; \ - default: \ - _InterlockedExchange((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ - break; \ - } - -#if defined(_M_X64) -/* This has to be an intptr_t for now. - gil_created() uses -1 as a sentinel value, if this returns - a uintptr_t it will do an unsigned compare and crash -*/ -inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t* value, int order) { - __int64 old; - switch (order) { - case _Py_memory_order_acquire: - { - do { - old = *value; - } while(_InterlockedCompareExchange64_HLEAcquire((volatile __int64*)value, old, old) != old); - break; - } - case _Py_memory_order_release: - { - do { - old = *value; - } while(_InterlockedCompareExchange64_HLERelease((volatile __int64*)value, old, old) != old); - break; - } - case _Py_memory_order_relaxed: - old = *value; - break; - default: - { - do { - old = *value; - } while(_InterlockedCompareExchange64((volatile __int64*)value, old, old) != old); - break; - } - } - return old; -} - -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \ - _Py_atomic_load_64bit_impl((volatile uintptr_t*)&((ATOMIC_VAL)->_value), (ORDER)) - -#else -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value) -#endif - -inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) { - long old; - switch (order) { - case _Py_memory_order_acquire: - { - do { - old = *value; - } while(_InterlockedCompareExchange_HLEAcquire((volatile long*)value, old, old) != old); - break; - } - case _Py_memory_order_release: - { - do { - old = *value; - } while(_InterlockedCompareExchange_HLERelease((volatile long*)value, old, old) != old); - break; - } - case _Py_memory_order_relaxed: - old = *value; - break; - default: - { - do { - old = *value; - } while(_InterlockedCompareExchange((volatile long*)value, old, old) != old); - break; - } - } - return old; -} - -#define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \ - _Py_atomic_load_32bit_impl((volatile int*)&((ATOMIC_VAL)->_value), (ORDER)) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - if (sizeof((ATOMIC_VAL)->_value) == 8) { \ - _Py_atomic_store_64bit((ATOMIC_VAL), NEW_VAL, ORDER) } else { \ - _Py_atomic_store_32bit((ATOMIC_VAL), NEW_VAL, ORDER) } - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - ( \ - sizeof((ATOMIC_VAL)->_value) == 8 ? \ - _Py_atomic_load_64bit((ATOMIC_VAL), ORDER) : \ - _Py_atomic_load_32bit((ATOMIC_VAL), ORDER) \ - ) -#elif defined(_M_ARM) || defined(_M_ARM64) -typedef enum _Py_memory_order { - _Py_memory_order_relaxed, - _Py_memory_order_acquire, - _Py_memory_order_release, - _Py_memory_order_acq_rel, - _Py_memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - volatile uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - volatile int _value; -} _Py_atomic_int; - - -#if defined(_M_ARM64) -#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \ - switch (ORDER) { \ - case _Py_memory_order_acquire: \ - _InterlockedExchange64_acq((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ - break; \ - case _Py_memory_order_release: \ - _InterlockedExchange64_rel((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ - break; \ - default: \ - _InterlockedExchange64((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ - break; \ - } -#else -#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) ((void)0); -#endif - -#define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \ - switch (ORDER) { \ - case _Py_memory_order_acquire: \ - _InterlockedExchange_acq((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ - break; \ - case _Py_memory_order_release: \ - _InterlockedExchange_rel((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ - break; \ - default: \ - _InterlockedExchange((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ - break; \ - } - -#if defined(_M_ARM64) -/* This has to be an intptr_t for now. - gil_created() uses -1 as a sentinel value, if this returns - a uintptr_t it will do an unsigned compare and crash -*/ -inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t* value, int order) { - uintptr_t old; - switch (order) { - case _Py_memory_order_acquire: - { - do { - old = *value; - } while(_InterlockedCompareExchange64_acq(value, old, old) != old); - break; - } - case _Py_memory_order_release: - { - do { - old = *value; - } while(_InterlockedCompareExchange64_rel(value, old, old) != old); - break; - } - case _Py_memory_order_relaxed: - old = *value; - break; - default: - { - do { - old = *value; - } while(_InterlockedCompareExchange64(value, old, old) != old); - break; - } - } - return old; -} - -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \ - _Py_atomic_load_64bit_impl((volatile uintptr_t*)&((ATOMIC_VAL)->_value), (ORDER)) - -#else -#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value) -#endif - -inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) { - int old; - switch (order) { - case _Py_memory_order_acquire: - { - do { - old = *value; - } while(_InterlockedCompareExchange_acq(value, old, old) != old); - break; - } - case _Py_memory_order_release: - { - do { - old = *value; - } while(_InterlockedCompareExchange_rel(value, old, old) != old); - break; - } - case _Py_memory_order_relaxed: - old = *value; - break; - default: - { - do { - old = *value; - } while(_InterlockedCompareExchange(value, old, old) != old); - break; - } - } - return old; -} - -#define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \ - _Py_atomic_load_32bit_impl((volatile int*)&((ATOMIC_VAL)->_value), (ORDER)) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - if (sizeof((ATOMIC_VAL)->_value) == 8) { \ - _Py_atomic_store_64bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } else { \ - _Py_atomic_store_32bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - ( \ - sizeof((ATOMIC_VAL)->_value) == 8 ? \ - _Py_atomic_load_64bit((ATOMIC_VAL), (ORDER)) : \ - _Py_atomic_load_32bit((ATOMIC_VAL), (ORDER)) \ - ) -#endif -#else /* !gcc x86 !_msc_ver */ -typedef enum _Py_memory_order { - _Py_memory_order_relaxed, - _Py_memory_order_acquire, - _Py_memory_order_release, - _Py_memory_order_acq_rel, - _Py_memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - int _value; -} _Py_atomic_int; -/* Fall back to other compilers and processors by assuming that simple - volatile accesses are atomic. This is false, so people should port - this. */ -#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0) -#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0) -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - ((ATOMIC_VAL)->_value = NEW_VAL) -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - ((ATOMIC_VAL)->_value) -#endif - -/* Standardized shortcuts. */ -#define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \ - _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_seq_cst) -#define _Py_atomic_load(ATOMIC_VAL) \ - _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_seq_cst) - -/* Python-local extensions */ - -#define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \ - _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_relaxed) -#define _Py_atomic_load_relaxed(ATOMIC_VAL) \ - _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_relaxed) - -#ifdef __cplusplus -} -#endif -#endif /* Py_ATOMIC_H */ diff --git a/Include/internal/pycore_atomic_funcs.h b/Include/internal/pycore_atomic_funcs.h deleted file mode 100644 index a708789cea733b..00000000000000 --- a/Include/internal/pycore_atomic_funcs.h +++ /dev/null @@ -1,94 +0,0 @@ -/* Atomic functions: similar to pycore_atomic.h, but don't need - to declare variables as atomic. - - Py_ssize_t type: - - * value = _Py_atomic_size_get(&var) - * _Py_atomic_size_set(&var, value) - - Use sequentially-consistent ordering (__ATOMIC_SEQ_CST memory order): - enforce total ordering with all other atomic functions. -*/ -#ifndef Py_ATOMIC_FUNC_H -#define Py_ATOMIC_FUNC_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#if defined(_MSC_VER) -# include // _InterlockedExchange() -#endif - - -// Use builtin atomic operations in GCC >= 4.7 and clang -#ifdef HAVE_BUILTIN_ATOMIC - -static inline Py_ssize_t _Py_atomic_size_get(Py_ssize_t *var) -{ - return __atomic_load_n(var, __ATOMIC_SEQ_CST); -} - -static inline void _Py_atomic_size_set(Py_ssize_t *var, Py_ssize_t value) -{ - __atomic_store_n(var, value, __ATOMIC_SEQ_CST); -} - -#elif defined(_MSC_VER) - -static inline Py_ssize_t _Py_atomic_size_get(Py_ssize_t *var) -{ -#if SIZEOF_VOID_P == 8 - Py_BUILD_ASSERT(sizeof(__int64) == sizeof(*var)); - volatile __int64 *volatile_var = (volatile __int64 *)var; - __int64 old; - do { - old = *volatile_var; - } while(_InterlockedCompareExchange64(volatile_var, old, old) != old); -#else - Py_BUILD_ASSERT(sizeof(long) == sizeof(*var)); - volatile long *volatile_var = (volatile long *)var; - long old; - do { - old = *volatile_var; - } while(_InterlockedCompareExchange(volatile_var, old, old) != old); -#endif - return old; -} - -static inline void _Py_atomic_size_set(Py_ssize_t *var, Py_ssize_t value) -{ -#if SIZEOF_VOID_P == 8 - Py_BUILD_ASSERT(sizeof(__int64) == sizeof(*var)); - volatile __int64 *volatile_var = (volatile __int64 *)var; - _InterlockedExchange64(volatile_var, value); -#else - Py_BUILD_ASSERT(sizeof(long) == sizeof(*var)); - volatile long *volatile_var = (volatile long *)var; - _InterlockedExchange(volatile_var, value); -#endif -} - -#else -// Fallback implementation using volatile - -static inline Py_ssize_t _Py_atomic_size_get(Py_ssize_t *var) -{ - volatile Py_ssize_t *volatile_var = (volatile Py_ssize_t *)var; - return *volatile_var; -} - -static inline void _Py_atomic_size_set(Py_ssize_t *var, Py_ssize_t value) -{ - volatile Py_ssize_t *volatile_var = (volatile Py_ssize_t *)var; - *volatile_var = value; -} -#endif - -#ifdef __cplusplus -} -#endif -#endif /* Py_ATOMIC_FUNC_H */ diff --git a/Include/internal/pycore_bitutils.h b/Include/internal/pycore_bitutils.h index e6bf61ef425bd8..50f69377523818 100644 --- a/Include/internal/pycore_bitutils.h +++ b/Include/internal/pycore_bitutils.h @@ -26,10 +26,10 @@ extern "C" { #endif #ifdef _MSC_VER - /* Get _byteswap_ushort(), _byteswap_ulong(), _byteswap_uint64() */ -# include +# include // _byteswap_uint64() #endif + static inline uint16_t _Py_bswap16(uint16_t word) { diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 28cf6fba4eeba2..573e10359b7bd2 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -40,6 +40,10 @@ extern "C" { #include "Python.h" +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + typedef struct { // List of bytes objects PyObject *list; @@ -314,4 +318,4 @@ _BlocksOutputBuffer_OnError(_BlocksOutputBuffer *buffer) #ifdef __cplusplus } #endif -#endif /* Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H */ \ No newline at end of file +#endif /* Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H */ diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index d36fa9569d64a5..94d421a9eb742a 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -8,39 +8,144 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +extern PyObject* _PyBytes_FormatEx( + const char *format, + Py_ssize_t format_len, + PyObject *args, + int use_bytearray); -/* Substring Search. +extern PyObject* _PyBytes_FromHex( + PyObject *string, + int use_bytearray); - Returns the index of the first occurrence of - a substring ("needle") in a larger text ("haystack"). - If the needle is not found, return -1. - If the needle is found, add offset to the index. -*/ +// Helper for PyBytes_DecodeEscape that detects invalid escape chars. +// Export for test_peg_generator. +PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t, + const char *, const char **); +/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, + x must be an iterable object. */ +extern PyObject* _PyBytes_Join(PyObject *sep, PyObject *x); + + +// Substring Search. +// +// Returns the index of the first occurrence of +// a substring ("needle") in a larger text ("haystack"). +// If the needle is not found, return -1. +// If the needle is found, add offset to the index. +// +// Export for 'mmap' shared extension. PyAPI_FUNC(Py_ssize_t) _PyBytes_Find(const char *haystack, Py_ssize_t len_haystack, const char *needle, Py_ssize_t len_needle, Py_ssize_t offset); -/* Same as above, but search right-to-left */ +// Same as above, but search right-to-left. +// Export for 'mmap' shared extension. PyAPI_FUNC(Py_ssize_t) _PyBytes_ReverseFind(const char *haystack, Py_ssize_t len_haystack, const char *needle, Py_ssize_t len_needle, Py_ssize_t offset); -/** Helper function to implement the repeat and inplace repeat methods on a buffer - * - * len_dest is assumed to be an integer multiple of len_src. - * If src equals dest, then assume the operation is inplace. - * - * This method repeately doubles the number of bytes copied to reduce - * the number of invocations of memcpy. - */ +// Helper function to implement the repeat and inplace repeat methods on a +// buffer. +// +// len_dest is assumed to be an integer multiple of len_src. +// If src equals dest, then assume the operation is inplace. +// +// This method repeately doubles the number of bytes copied to reduce +// the number of invocations of memcpy. +// +// Export for 'array' shared extension. PyAPI_FUNC(void) _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); +/* --- _PyBytesWriter ----------------------------------------------------- */ + +/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". + A _PyBytesWriter variable must be declared at the end of variables in a + function to optimize the memory allocation on the stack. */ +typedef struct { + /* bytes, bytearray or NULL (when the small buffer is used) */ + PyObject *buffer; + + /* Number of allocated size. */ + Py_ssize_t allocated; + + /* Minimum number of allocated bytes, + incremented by _PyBytesWriter_Prepare() */ + Py_ssize_t min_size; + + /* If non-zero, use a bytearray instead of a bytes object for buffer. */ + int use_bytearray; + + /* If non-zero, overallocate the buffer (default: 0). + This flag must be zero if use_bytearray is non-zero. */ + int overallocate; + + /* Stack buffer */ + int use_small_buffer; + char small_buffer[512]; +} _PyBytesWriter; + +/* Initialize a bytes writer + + By default, the overallocation is disabled. Set the overallocate attribute + to control the allocation of the buffer. + + Export _PyBytesWriter API for '_pickle' shared extension. */ +PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); + +/* Get the buffer content and reset the writer. + Return a bytes object, or a bytearray object if use_bytearray is non-zero. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, + void *str); + +/* Deallocate memory of a writer (clear its internal buffer). */ +PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); + +/* Allocate the buffer to write size bytes. + Return the pointer to the beginning of buffer data. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, + Py_ssize_t size); + +/* Ensure that the buffer is large enough to write *size* bytes. + Add size to the writer minimum size (min_size attribute). + + str is the current pointer inside the buffer. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, + void *str, + Py_ssize_t size); + +/* Resize the buffer to make it larger. + The new buffer may be larger than size bytes because of overallocation. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. + + Note: size must be greater than the number of allocated bytes in the writer. + + This function doesn't use the writer minimum size (min_size attribute). + + See also _PyBytesWriter_Prepare(). + */ +PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, + void *str, + Py_ssize_t size); + +/* Write bytes. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, + void *str, + const void *bytes, + Py_ssize_t size); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 5d9342b562b002..c92028a01299e2 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -8,31 +8,107 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_identifier.h" // _Py_Identifier #include "pycore_pystate.h" // _PyThreadState_GET() -PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend( +/* Suggested size (number of positional arguments) for arrays of PyObject* + allocated on a C stack to avoid allocating memory on the heap memory. Such + array is used to pass positional arguments to call functions of the + PyObject_Vectorcall() family. + + The size is chosen to not abuse the C stack and so limit the risk of stack + overflow. The size is also chosen to allow using the small stack for most + function calls of the Python standard library. On 64-bit CPU, it allocates + 40 bytes on the stack. */ +#define _PY_FASTCALL_SMALL_STACK 5 + + +// Export for 'math' shared extension, used via _PyObject_VectorcallTstate() +// static inline function. +PyAPI_FUNC(PyObject*) _Py_CheckFunctionResult( + PyThreadState *tstate, + PyObject *callable, + PyObject *result, + const char *where); + +extern PyObject* _PyObject_Call_Prepend( PyThreadState *tstate, PyObject *callable, PyObject *obj, PyObject *args, PyObject *kwargs); -PyAPI_FUNC(PyObject *) _PyObject_FastCallDictTstate( +extern PyObject* _PyObject_VectorcallDictTstate( PyThreadState *tstate, PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwargs); -PyAPI_FUNC(PyObject *) _PyObject_Call( +extern PyObject* _PyObject_Call( PyThreadState *tstate, PyObject *callable, PyObject *args, PyObject *kwargs); extern PyObject * _PyObject_CallMethodFormat( - PyThreadState *tstate, PyObject *callable, const char *format, ...); + PyThreadState *tstate, + PyObject *callable, + const char *format, + ...); + +// Export for 'array' shared extension +PyAPI_FUNC(PyObject*) _PyObject_CallMethod( + PyObject *obj, + PyObject *name, + const char *format, ...); + +extern PyObject* _PyObject_CallMethodIdObjArgs( + PyObject *obj, + _Py_Identifier *name, + ...); + +static inline PyObject * +_PyObject_VectorcallMethodId( + _Py_Identifier *name, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ + if (!oname) { + return _Py_NULL; + } + return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); +} + +static inline PyObject * +_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) +{ + size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); +} + +static inline PyObject * +_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) +{ + PyObject *args[2] = {self, arg}; + size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; + assert(arg != NULL); + return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); +} + + +/* === Vectorcall protocol (PEP 590) ============================= */ +// Call callable using tp_call. Arguments are like PyObject_Vectorcall(), +// except that nargs is plainly the number of arguments without flags. +// +// Export for 'math' shared extension, used via _PyObject_VectorcallTstate() +// static inline function. +PyAPI_FUNC(PyObject*) _PyObject_MakeTpCall( + PyThreadState *tstate, + PyObject *callable, + PyObject *const *args, Py_ssize_t nargs, + PyObject *keywords); // Static inline variant of public PyVectorcall_Function(). static inline vectorcallfunc @@ -109,23 +185,19 @@ _PyObject_CallNoArgs(PyObject *func) { } -static inline PyObject * -_PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const *args, Py_ssize_t nargs) -{ - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, func); - return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); -} - -PyObject *const * +extern PyObject *const * _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames); -void -_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, +extern void _PyStack_UnpackDict_Free( + PyObject *const *stack, + Py_ssize_t nargs, PyObject *kwnames); -void _PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames); +extern void _PyStack_UnpackDict_FreeNoDecRef( + PyObject *const *stack, + PyObject *kwnames); #ifdef __cplusplus } diff --git a/Include/internal/pycore_capsule.h b/Include/internal/pycore_capsule.h new file mode 100644 index 00000000000000..aa2c67f3a8f002 --- /dev/null +++ b/Include/internal/pycore_capsule.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERNAL_PYCAPSULE_H +#define Py_INTERNAL_PYCAPSULE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Export for '_socket' shared extension +PyAPI_FUNC(int) _PyCapsule_SetTraverse(PyObject *op, traverseproc traverse_func, inquiry clear_func); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYCAPSULE_H */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 3c8b368bd2af4e..64fb4034669e19 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -8,27 +8,54 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "dynamic_annotations.h" // _Py_ANNOTATE_RWLOCK_CREATE + +#include "pycore_interp.h" // PyInterpreterState.eval_frame +#include "pycore_pystate.h" // _PyThreadState_GET() + /* Forward declarations */ struct pyruntimestate; struct _ceval_runtime_state; +// Export for '_lsprof' shared extension +PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); + +extern int _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); + +extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); + +// Helper to look up a builtin object +// Export for 'array' shared extension +PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); + +extern PyObject* _PyEval_GetBuiltinId(_Py_Identifier *); + +extern void _PyEval_SetSwitchInterval(unsigned long microseconds); +extern unsigned long _PyEval_GetSwitchInterval(void); + +// Export for '_queue' shared extension +PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); + #ifndef Py_DEFAULT_RECURSION_LIMIT # define Py_DEFAULT_RECURSION_LIMIT 1000 #endif -#include "pycore_interp.h" // PyInterpreterState.eval_frame -#include "pycore_pystate.h" // _PyThreadState_GET() +extern void _Py_FinishPendingCalls(PyThreadState *tstate); +extern void _PyEval_InitState(PyInterpreterState *); +extern void _PyEval_SignalReceived(PyInterpreterState *interp); +// bitwise flags: +#define _Py_PENDING_MAINTHREADONLY 1 +#define _Py_PENDING_RAWFREE 2 -extern void _Py_FinishPendingCalls(PyThreadState *tstate); -extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); -extern void _PyEval_FiniState(struct _ceval_state *ceval); -PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyEval_AddPendingCall( PyInterpreterState *interp, - int (*func)(void *), - void *arg); -PyAPI_FUNC(void) _PyEval_SignalAsyncExc(PyInterpreterState *interp); + _Py_pending_call_func func, + void *arg, + int flags); + +extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); #endif @@ -73,6 +100,7 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); +extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -100,8 +128,7 @@ extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_AcquireLock(PyThreadState *tstate); -extern void _PyEval_ReleaseLock(PyThreadState *tstate); -extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *); +extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); extern void _PyEval_DeactivateOpCache(void); @@ -121,6 +148,8 @@ static inline int _Py_MakeRecCheck(PyThreadState *tstate) { } #endif +// Export for '_json' shared extension, used via _Py_EnterRecursiveCall() +// static inline function. PyAPI_FUNC(int) _Py_CheckRecursiveCall( PyThreadState *tstate, const char *where); @@ -151,8 +180,57 @@ extern struct _PyInterpreterFrame* _PyEval_GetFrame(void); extern PyObject* _Py_MakeCoro(PyFunctionObject *func); +/* Handle signals, pending calls, GIL drop request + and asynchronous exception */ extern int _Py_HandlePending(PyThreadState *tstate); +extern PyObject * _PyEval_GetFrameLocals(void); + +extern const binaryfunc _PyEval_BinaryOps[]; +int _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); +int _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); +int _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); +void _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); +void _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); +void _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); +void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); +PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); +int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); +void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); + + +#define _PY_GIL_DROP_REQUEST_BIT 0 +#define _PY_SIGNALS_PENDING_BIT 1 +#define _PY_CALLS_TO_DO_BIT 2 +#define _PY_ASYNC_EXCEPTION_BIT 3 +#define _PY_GC_SCHEDULED_BIT 4 + +/* Reserve a few bits for future use */ +#define _PY_EVAL_EVENTS_BITS 8 +#define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) + +static inline void +_Py_set_eval_breaker_bit(PyInterpreterState *interp, uint32_t bit, uint32_t set) +{ + assert(set == 0 || set == 1); + uintptr_t to_set = set << bit; + uintptr_t mask = ((uintptr_t)1) << bit; + uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); + if ((old & mask) == to_set) { + return; + } + uintptr_t new; + do { + new = (old & ~mask) | to_set; + } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); +} + +static inline bool +_Py_eval_breaker_bit_is_set(PyInterpreterState *interp, int32_t bit) +{ + return _Py_atomic_load_uintptr_relaxed(&interp->ceval.eval_breaker) & (((uintptr_t)1) << bit); +} #ifdef __cplusplus diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index b352801673c40a..28738980eb49be 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -8,11 +8,27 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - -#include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_lock.h" // PyMutex #include "pycore_gil.h" // struct _gil_runtime_state +typedef int (*_Py_pending_call_func)(void *); + +struct _pending_calls { + int busy; + PyMutex mutex; + /* Request for running pending calls. */ + int32_t calls_to_do; +#define NPENDINGCALLS 32 + struct _pending_call { + _Py_pending_call_func func; + void *arg; + int flags; + } calls[NPENDINGCALLS]; + int first; + int last; +}; + typedef enum { PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized @@ -40,15 +56,13 @@ struct _ceval_runtime_state { struct code_arena_st *code_arena; struct trampoline_api_st trampoline_api; FILE *map_file; + Py_ssize_t persist_after_fork; #else int _not_used; #endif } perf; - /* Request for checking signals. It is shared by all interpreters (see - bpo-40513). Any thread of any interpreter can receive a signal, but only - the main thread of the main interpreter can handle signals: see - _Py_ThreadCanHandleSignals(). */ - _Py_atomic_int signals_pending; + /* Pending calls to be made only on the main thread. */ + struct _pending_calls pending_mainthread; }; #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -56,41 +70,24 @@ struct _ceval_runtime_state { { \ .status = PERF_STATUS_NO_INIT, \ .extra_code_index = -1, \ + .persist_after_fork = 0, \ } #else # define _PyEval_RUNTIME_PERF_INIT {0} #endif -struct _pending_calls { - int busy; - PyThread_type_lock lock; - /* Request for running pending calls. */ - _Py_atomic_int calls_to_do; - /* Request for looking at the `async_exc` field of the current - thread state. - Guarded by the GIL. */ - int async_exc; -#define NPENDINGCALLS 32 - struct { - int (*func)(void *); - void *arg; - } calls[NPENDINGCALLS]; - int first; - int last; -}; - struct _ceval_state { + /* This single variable consolidates all requests to break out of + * the fast path in the eval loop. + * It is by far the hottest field in this struct and + * should be placed at the beginning. */ + uintptr_t eval_breaker; + /* Avoid false sharing */ + int64_t padding[7]; int recursion_limit; struct _gil_runtime_state *gil; int own_gil; - /* This single variable consolidates all requests to break out of - the fast path in the eval loop. */ - _Py_atomic_int eval_breaker; - /* Request for dropping the GIL */ - _Py_atomic_int gil_drop_request; - /* The GC is ready to be executed */ - _Py_atomic_int gc_scheduled; struct _pending_calls pending; }; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 75a23f3f5af560..73df6c3568ffe0 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -4,6 +4,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #define CODE_MAX_WATCHERS 8 /* PEP 659 @@ -19,9 +23,9 @@ extern "C" { typedef struct { uint16_t counter; - uint16_t index; uint16_t module_keys_version; uint16_t builtin_keys_version; + uint16_t index; } _PyLoadGlobalCache; #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) @@ -101,6 +105,13 @@ typedef struct { #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache) +typedef struct { + uint16_t counter; + uint16_t version[2]; +} _PyToBoolCache; + +#define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) + // Borrowed references to common callables: struct callable_cache { PyObject *isinstance; @@ -196,8 +207,8 @@ struct _PyCodeConstructor { // back to a regular function signature. Regardless, this approach // wouldn't be appropriate if this weren't a strictly internal API. // (See the comments in https://github.com/python/cpython/pull/26258.) -PyAPI_FUNC(int) _PyCode_Validate(struct _PyCodeConstructor *); -PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *); +extern int _PyCode_Validate(struct _PyCodeConstructor *); +extern PyCodeObject* _PyCode_New(struct _PyCodeConstructor *); /* Private API */ @@ -222,6 +233,11 @@ extern void _PyLineTable_InitAddressRange( extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); +/** API for executors */ +extern void _PyCode_Clear_Executors(PyCodeObject *code); + +#define ENABLE_SPECIALIZATION 1 + /* Specialization functions */ extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, @@ -237,7 +253,7 @@ extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr); extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, - int nargs, PyObject *kwnames); + int nargs); extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals); extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, @@ -246,6 +262,7 @@ extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr); /* Finalizer function for static codeobjects used in deepfreeze.py */ extern void _PyStaticCode_Fini(PyCodeObject *co); @@ -254,19 +271,32 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #ifdef Py_STATS +#include "pycore_bitutils.h" // _Py_bit_length -#define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0) -#define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0) -#define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0) -#define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0) -#define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0) +#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) +#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) +#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) +#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) +#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) #define OBJECT_STAT_INC_COND(name, cond) \ - do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0) -#define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0) + do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) +#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ - do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0) - -// Used by the _opcode extension which is built as a shared library + do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) +#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) +#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) +#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) +#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) +#define OPT_HIST(length, name) \ + do { \ + if (_Py_stats) { \ + int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ + bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ + _Py_stats->optimization_stats.name[bucket]++; \ + } \ + } while (0) + +// Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #else @@ -278,6 +308,11 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) +#define GC_STAT_ADD(gen, name, n) ((void)0) +#define OPT_STAT_INC(name) ((void)0) +#define UOP_STAT_INC(opname, name) ((void)0) +#define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) +#define OPT_HIST(length, name) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. @@ -359,27 +394,29 @@ write_varint(uint8_t *ptr, unsigned int val) val >>= 6; written++; } - *ptr = val; + *ptr = (uint8_t)val; return written; } static inline int write_signed_varint(uint8_t *ptr, int val) { + unsigned int uval; if (val < 0) { - val = ((-val)<<1) | 1; + // (unsigned int)(-val) has an undefined behavior for INT_MIN + uval = ((0 - (unsigned int)val) << 1) | 1; } else { - val = val << 1; + uval = (unsigned int)val << 1; } - return write_varint(ptr, val); + return write_varint(ptr, uval); } static inline int write_location_entry_start(uint8_t *ptr, int code, int length) { assert((code & 15) == code); - *ptr = 128 | (code << 3) | (length - 1); + *ptr = 128 | (uint8_t)(code << 3) | (uint8_t)(length - 1); return 1; } @@ -419,9 +456,9 @@ write_location_entry_start(uint8_t *ptr, int code, int length) static inline uint16_t -adaptive_counter_bits(int value, int backoff) { - return (value << ADAPTIVE_BACKOFF_BITS) | - (backoff & ((1< MAX_BACKOFF_VALUE) { backoff = MAX_BACKOFF_VALUE; } - unsigned int value = (1 << backoff) - 1; + uint16_t value = (uint16_t)(1 << backoff) - 1; return adaptive_counter_bits(value, backoff); } -/* Line array cache for tracing */ - -typedef struct _PyShimCodeDef { - const uint8_t *code; - int codelen; - int stacksize; - const char *cname; -} _PyShimCodeDef; - -extern PyCodeObject * -_Py_MakeShimCode(const _PyShimCodeDef *code); - -extern uint32_t _Py_next_func_version; - - /* Comparison bit masks. */ /* Note this evaluates its arguments twice each */ @@ -487,6 +509,7 @@ extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset); +extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); #ifdef __cplusplus } diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h new file mode 100644 index 00000000000000..a2a7151d50ade7 --- /dev/null +++ b/Include/internal/pycore_codecs.h @@ -0,0 +1,55 @@ +#ifndef Py_INTERNAL_CODECS_H +#define Py_INTERNAL_CODECS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern PyObject* _PyCodec_Lookup(const char *encoding); + +/* Text codec specific encoding and decoding API. + + Checks the encoding against a list of codecs which do not + implement a str<->bytes encoding before attempting the + operation. + + Please note that these APIs are internal and should not + be used in Python C extensions. + + XXX (ncoghlan): should we make these, or something like them, public + in Python 3.5+? + + */ +extern PyObject* _PyCodec_LookupTextEncoding( + const char *encoding, + const char *alternate_command); + +extern PyObject* _PyCodec_EncodeText( + PyObject *object, + const char *encoding, + const char *errors); + +extern PyObject* _PyCodec_DecodeText( + PyObject *object, + const char *encoding, + const char *errors); + +/* These two aren't actually text encoding specific, but _io.TextIOWrapper + * is the only current API consumer. + */ +extern PyObject* _PyCodecInfo_GetIncrementalDecoder( + PyObject *codec_info, + const char *errors); + +extern PyObject* _PyCodecInfo_GetIncrementalEncoder( + PyObject *codec_info, + const char *errors); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CODECS_H */ diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 499f55f3e276be..e5870759ba74f1 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -11,7 +11,7 @@ extern "C" { struct _arena; // Type defined in pycore_pyarena.h struct _mod; // Type defined in pycore_ast.h -// Export the symbol for test_peg_generator (built as a library) +// Export for 'test_peg_generator' shared extension PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( struct _mod *mod, PyObject *filename, @@ -19,23 +19,24 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( int optimize, struct _arena *arena); -static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; - -typedef struct { - int optimize; - int ff_features; +/* AST optimizations */ +extern int _PyCompile_AstOptimize( + struct _mod *mod, + PyObject *filename, + PyCompilerFlags *flags, + int optimize, + struct _arena *arena); - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ -} _PyASTOptimizeState; +static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; extern int _PyAST_Optimize( struct _mod *, struct _arena *arena, - _PyASTOptimizeState *state); + int optimize, + int ff_features); typedef struct { - int h_offset; + int h_label; int h_startdepth; int h_preserve_lasti; } _PyCompile_ExceptHandlerInfo; @@ -45,6 +46,10 @@ typedef struct { int i_oparg; _PyCompilerSrcLocation i_loc; _PyCompile_ExceptHandlerInfo i_except_handler_info; + + /* Used by the assembler */ + int i_target; + int i_offset; } _PyCompile_Instruction; typedef struct { @@ -57,6 +62,11 @@ typedef struct { int s_next_free_label; /* next free label id */ } _PyCompile_InstructionSequence; +int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl); +int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, + int opcode, int oparg, + _PyCompilerSrcLocation loc); + typedef struct { PyObject *u_name; PyObject *u_qualname; /* dot-separated qualified name (lazy) */ @@ -92,10 +102,26 @@ int _PyCompile_EnsureArrayLargeEnough( int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); -int _PyCompile_InstrSize(int opcode, int oparg); + +// Export for '_opcode' extention module +PyAPI_FUNC(int) _PyCompile_OpcodeIsValid(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasArg(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasConst(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasName(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasJump(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasFree(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasLocal(int opcode); +PyAPI_FUNC(int) _PyCompile_OpcodeHasExc(int opcode); + +PyAPI_FUNC(PyObject*) _PyCompile_GetUnaryIntrinsicName(int index); +PyAPI_FUNC(PyObject*) _PyCompile_GetBinaryIntrinsicName(int index); /* Access compiler internals for unit testing */ +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyCompile_CleanDoc(PyObject *doc); + +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( PyObject *ast, PyObject *filename, @@ -103,10 +129,13 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( int optimize, int compile_mode); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, - PyObject *consts); + PyObject *consts, + int nlocals); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyCodeObject*) _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, PyObject *instructions); diff --git a/Include/internal/pycore_complexobject.h b/Include/internal/pycore_complexobject.h new file mode 100644 index 00000000000000..54713536eedc46 --- /dev/null +++ b/Include/internal/pycore_complexobject.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_COMPLEXOBJECT_H +#define Py_INTERNAL_COMPLEXOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_unicodeobject.h" // _PyUnicodeWriter + +/* Format the object based on the format_spec, as defined in PEP 3101 + (Advanced String Formatting). */ +extern int _PyComplex_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_COMPLEXOBJECT_H diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 981c962bf7dfdf..34c21aaad43197 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -5,14 +5,8 @@ # error "this header requires Py_BUILD_CORE define" #endif -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include /* _POSIX_THREADS */ -# endif -#endif +#include "pycore_pythread.h" // _POSIX_THREADS + #ifdef _POSIX_THREADS /* @@ -21,7 +15,7 @@ #define Py_HAVE_CONDVAR #ifdef HAVE_PTHREAD_H -# include +# include // pthread_mutex_t #endif #define PyMUTEX_T pthread_mutex_t @@ -38,7 +32,7 @@ /* include windows if it hasn't been done before */ #define WIN32_LEAN_AND_MEAN -#include +#include // CRITICAL_SECTION /* options */ /* non-emulated condition variables are provided for those that want diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 52dfe3ef233874..ec884e9e0f55a9 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,7 +5,7 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_hamt.h" /* PyHamtObject */ +#include "pycore_hamt.h" // PyHamtObject extern PyTypeObject _PyContextTokenMissing_Type; @@ -68,4 +68,9 @@ struct _pycontexttokenobject { }; +// _testinternalcapi.hamt() used by tests. +// Export for '_testcapi' shared extension +PyAPI_FUNC(PyObject*) _PyContext_NewHamtForTests(void); + + #endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h new file mode 100644 index 00000000000000..bf2bbfffc38bd0 --- /dev/null +++ b/Include/internal/pycore_critical_section.h @@ -0,0 +1,242 @@ +#ifndef Py_INTERNAL_CRITICAL_SECTION_H +#define Py_INTERNAL_CRITICAL_SECTION_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_lock.h" // PyMutex +#include "pycore_pystate.h" // _PyThreadState_GET() +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Implementation of Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// Straightforward per-object locking could introduce deadlocks that were not +// present when running with the GIL. Threads may hold locks for multiple +// objects simultaneously because Python operations can nest. If threads were +// to acquire the same locks in different orders, they would deadlock. +// +// One way to avoid deadlocks is to allow threads to hold only the lock (or +// locks) for a single operation at a time (typically a single lock, but some +// operations involve two locks). When a thread begins a nested operation it +// could suspend the locks for any outer operation: before beginning the nested +// operation, the locks for the outer operation are released and when the +// nested operation completes, the locks for the outer operation are +// reacquired. +// +// To improve performance, this API uses a variation of the above scheme. +// Instead of immediately suspending locks any time a nested operation begins, +// locks are only suspended if the thread would block. This reduces the number +// of lock acquisitions and releases for nested operations, while still +// avoiding deadlocks. +// +// Additionally, the locks for any active operation are suspended around +// other potentially blocking operations, such as I/O. This is because the +// interaction between locks and blocking operations can lead to deadlocks in +// the same way as the interaction between multiple locks. +// +// Each thread's critical sections and their corresponding locks are tracked in +// a stack in `PyThreadState.critical_section`. When a thread calls +// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when +// waiting to acquire a lock, the thread suspends all of its active critical +// sections, temporarily releasing the associated locks. When the thread calls +// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) +// critical section by reacquiring the associated lock or locks. See +// `_PyCriticalSection_Resume()`. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + + +// Tagged pointers to critical sections use the two least significant bits to +// mark if the pointed-to critical section is inactive and whether it is a +// _PyCriticalSection2 object. +#define _Py_CRITICAL_SECTION_INACTIVE 0x1 +#define _Py_CRITICAL_SECTION_TWO_MUTEXES 0x2 +#define _Py_CRITICAL_SECTION_MASK 0x3 + +#ifdef Py_GIL_DISABLED +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + _PyCriticalSection _cs; \ + _PyCriticalSection_Begin(&_cs, &_PyObject_CAST(op)->ob_mutex) + +# define Py_END_CRITICAL_SECTION() \ + _PyCriticalSection_End(&_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + _PyCriticalSection2 _cs2; \ + _PyCriticalSection2_Begin(&_cs2, &_PyObject_CAST(a)->ob_mutex, &_PyObject_CAST(b)->ob_mutex) + +# define Py_END_CRITICAL_SECTION2() \ + _PyCriticalSection2_End(&_cs2); \ + } +#else /* !Py_GIL_DISABLED */ +// The critical section APIs are no-ops with the GIL. +# define Py_BEGIN_CRITICAL_SECTION(op) +# define Py_END_CRITICAL_SECTION() +# define Py_BEGIN_CRITICAL_SECTION2(a, b) +# define Py_END_CRITICAL_SECTION2() +#endif /* !Py_GIL_DISABLED */ + +typedef struct { + // Tagged pointer to an outer active critical section (or 0). + // The two least-significant-bits indicate whether the pointed-to critical + // section is inactive and whether it is a _PyCriticalSection2 object. + uintptr_t prev; + + // Mutex used to protect critical section + PyMutex *mutex; +} _PyCriticalSection; + +// A critical section protected by two mutexes. Use +// _PyCriticalSection2_Begin and _PyCriticalSection2_End. +typedef struct { + _PyCriticalSection base; + + PyMutex *mutex2; +} _PyCriticalSection2; + +static inline int +_PyCriticalSection_IsActive(uintptr_t tag) +{ + return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0; +} + +// Resumes the top-most critical section. +PyAPI_FUNC(void) +_PyCriticalSection_Resume(PyThreadState *tstate); + +// (private) slow path for locking the mutex +PyAPI_FUNC(void) +_PyCriticalSection_BeginSlow(_PyCriticalSection *c, PyMutex *m); + +PyAPI_FUNC(void) +_PyCriticalSection2_BeginSlow(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, + int is_m1_locked); + +static inline void +_PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m) +{ + if (PyMutex_LockFast(&m->v)) { + PyThreadState *tstate = _PyThreadState_GET(); + c->mutex = m; + c->prev = tstate->critical_section; + tstate->critical_section = (uintptr_t)c; + } + else { + _PyCriticalSection_BeginSlow(c, m); + } +} + +// Removes the top-most critical section from the thread's stack of critical +// sections. If the new top-most critical section is inactive, then it is +// resumed. +static inline void +_PyCriticalSection_Pop(_PyCriticalSection *c) +{ + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t prev = c->prev; + tstate->critical_section = prev; + + if ((prev & _Py_CRITICAL_SECTION_INACTIVE) != 0) { + _PyCriticalSection_Resume(tstate); + } +} + +static inline void +_PyCriticalSection_End(_PyCriticalSection *c) +{ + PyMutex_Unlock(c->mutex); + _PyCriticalSection_Pop(c); +} + +static inline void +_PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +{ + if (m1 == m2) { + // If the two mutex arguments are the same, treat this as a critical + // section with a single mutex. + c->mutex2 = NULL; + _PyCriticalSection_Begin(&c->base, m1); + return; + } + + if ((uintptr_t)m2 < (uintptr_t)m1) { + // Sort the mutexes so that the lower address is locked first. + // The exact order does not matter, but we need to acquire the mutexes + // in a consistent order to avoid lock ordering deadlocks. + PyMutex *tmp = m1; + m1 = m2; + m2 = tmp; + } + + if (PyMutex_LockFast(&m1->v)) { + if (PyMutex_LockFast(&m2->v)) { + PyThreadState *tstate = _PyThreadState_GET(); + c->base.mutex = m1; + c->mutex2 = m2; + c->base.prev = tstate->critical_section; + + uintptr_t p = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES; + tstate->critical_section = p; + } + else { + _PyCriticalSection2_BeginSlow(c, m1, m2, 1); + } + } + else { + _PyCriticalSection2_BeginSlow(c, m1, m2, 0); + } +} + +static inline void +_PyCriticalSection2_End(_PyCriticalSection2 *c) +{ + if (c->mutex2) { + PyMutex_Unlock(c->mutex2); + } + PyMutex_Unlock(c->base.mutex); + _PyCriticalSection_Pop(&c->base); +} + +PyAPI_FUNC(void) +_PyCriticalSection_SuspendAll(PyThreadState *tstate); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CRITICAL_SECTION_H */ diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h new file mode 100644 index 00000000000000..2e6d09a49f95d3 --- /dev/null +++ b/Include/internal/pycore_crossinterp.h @@ -0,0 +1,282 @@ +#ifndef Py_INTERNAL_CROSSINTERP_H +#define Py_INTERNAL_CROSSINTERP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_lock.h" // PyMutex +#include "pycore_pyerrors.h" + + +/***************************/ +/* cross-interpreter calls */ +/***************************/ + +typedef int (*_Py_simple_func)(void *); +extern int _Py_CallInInterpreter( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); +extern int _Py_CallInInterpreterAndRawFree( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); + + +/**************************/ +/* cross-interpreter data */ +/**************************/ + +typedef struct _xid _PyCrossInterpreterData; +typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef void (*xid_freefunc)(void *); + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass safely between interpreters in the same process. +struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case the code that sets the field, + // likely a registered "crossinterpdatafunc", is responsible for + // ensuring it owns the reference (i.e. incref). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. Note that IDs are never re-used, so + // each one will always correspond to a specific interpreter + // (whether still alive or not). + int64_t interpid; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + xid_newobjectfunc new_object; + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + xid_freefunc free; +}; + +PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); +PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); + + +/* defining cross-interpreter data */ + +PyAPI_FUNC(void) _PyCrossInterpreterData_Init( + _PyCrossInterpreterData *data, + PyInterpreterState *interp, void *shared, PyObject *obj, + xid_newobjectfunc new_object); +PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( + _PyCrossInterpreterData *, + PyInterpreterState *interp, const size_t, PyObject *, + xid_newobjectfunc); +PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( + PyInterpreterState *, _PyCrossInterpreterData *); + + +/* using cross-interpreter data */ + +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); + + +/* cross-interpreter data registry */ + +// For now we use a global registry of shareable classes. An +// alternative would be to add a tp_* slot for a class's +// crossinterpdatafunc. It would be simpler and more efficient. + +typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, + _PyCrossInterpreterData *); + +struct _xidregitem; + +struct _xidregitem { + struct _xidregitem *prev; + struct _xidregitem *next; + /* This can be a dangling pointer, but only if weakref is set. */ + PyTypeObject *cls; + /* This is NULL for builtin types. */ + PyObject *weakref; + size_t refcount; + crossinterpdatafunc getdata; +}; + +struct _xidregistry { + int global; /* builtin types or heap types */ + int initialized; + PyMutex mutex; + struct _xidregitem *head; +}; + +PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + + +/*****************************/ +/* runtime state & lifecycle */ +/*****************************/ + +struct _xi_runtime_state { + // builtin types + // XXX Remove this field once we have a tp_* slot. + struct _xidregistry registry; +}; + +struct _xi_state { + // heap types + // XXX Remove this field once we have a tp_* slot. + struct _xidregistry registry; + + // heap types + PyObject *PyExc_NotShareableError; +}; + +extern PyStatus _PyXI_Init(PyInterpreterState *interp); +extern void _PyXI_Fini(PyInterpreterState *interp); + + +/***************************/ +/* short-term data sharing */ +/***************************/ + +// Ultimately we'd like to preserve enough information about the +// exception and traceback that we could re-constitute (or at least +// simulate, a la traceback.TracebackException), and even chain, a copy +// of the exception in the calling interpreter. + +typedef struct _excinfo { + struct _excinfo_type { + PyTypeObject *builtin; + const char *name; + const char *qualname; + const char *module; + } type; + const char *msg; +} _PyXI_excinfo; + + +typedef enum error_code { + _PyXI_ERR_NO_ERROR = 0, + _PyXI_ERR_UNCAUGHT_EXCEPTION = -1, + _PyXI_ERR_OTHER = -2, + _PyXI_ERR_NO_MEMORY = -3, + _PyXI_ERR_ALREADY_RUNNING = -4, + _PyXI_ERR_MAIN_NS_FAILURE = -5, + _PyXI_ERR_APPLY_NS_FAILURE = -6, + _PyXI_ERR_NOT_SHAREABLE = -7, +} _PyXI_errcode; + + +typedef struct _sharedexception { + // The originating interpreter. + PyInterpreterState *interp; + // The kind of error to propagate. + _PyXI_errcode code; + // The exception information to propagate, if applicable. + // This is populated only for some error codes, + // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. + _PyXI_excinfo uncaught; +} _PyXI_error; + +PyAPI_FUNC(PyObject *) _PyXI_ApplyError(_PyXI_error *err); + + +typedef struct xi_session _PyXI_session; +typedef struct _sharedns _PyXI_namespace; + +PyAPI_FUNC(void) _PyXI_FreeNamespace(_PyXI_namespace *ns); +PyAPI_FUNC(_PyXI_namespace *) _PyXI_NamespaceFromNames(PyObject *names); +PyAPI_FUNC(int) _PyXI_FillNamespaceFromDict( + _PyXI_namespace *ns, + PyObject *nsobj, + _PyXI_session *session); +PyAPI_FUNC(int) _PyXI_ApplyNamespace( + _PyXI_namespace *ns, + PyObject *nsobj, + PyObject *dflt); + + +// A cross-interpreter session involves entering an interpreter +// (_PyXI_Enter()), doing some work with it, and finally exiting +// that interpreter (_PyXI_Exit()). +// +// At the boundaries of the session, both entering and exiting, +// data may be exchanged between the previous interpreter and the +// target one in a thread-safe way that does not violate the +// isolation between interpreters. This includes setting objects +// in the target's __main__ module on the way in, and capturing +// uncaught exceptions on the way out. +struct xi_session { + // Once a session has been entered, this is the tstate that was + // current before the session. If it is different from cur_tstate + // then we must have switched interpreters. Either way, this will + // be the current tstate once we exit the session. + PyThreadState *prev_tstate; + // Once a session has been entered, this is the current tstate. + // It must be current when the session exits. + PyThreadState *init_tstate; + // This is true if init_tstate needs cleanup during exit. + int own_init_tstate; + + // This is true if, while entering the session, init_thread took + // "ownership" of the interpreter's __main__ module. This means + // it is the only thread that is allowed to run code there. + // (Caveat: for now, users may still run exec() against the + // __main__ module's dict, though that isn't advisable.) + int running; + // This is a cached reference to the __dict__ of the entered + // interpreter's __main__ module. It is looked up when at the + // beginning of the session as a convenience. + PyObject *main_ns; + + // This is set if the interpreter is entered and raised an exception + // that needs to be handled in some special way during exit. + _PyXI_errcode *error_override; + // This is set if exit captured an exception to propagate. + _PyXI_error *error; + + // -- pre-allocated memory -- + _PyXI_error _error; + _PyXI_errcode _error_override; +}; + +PyAPI_FUNC(int) _PyXI_Enter( + _PyXI_session *session, + PyInterpreterState *interp, + PyObject *nsupdates); +PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session); + +PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); +PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CROSSINTERP_H */ diff --git a/Include/internal/pycore_descrobject.h b/Include/internal/pycore_descrobject.h index 76378569df90e3..3cec59a68a3d2b 100644 --- a/Include/internal/pycore_descrobject.h +++ b/Include/internal/pycore_descrobject.h @@ -20,6 +20,8 @@ typedef struct { typedef propertyobject _PyPropertyObject; +extern PyTypeObject _PyMethodWrapper_Type; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6253e0841ad349..d96870e9197bbf 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -9,8 +9,62 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_dict_state.h" -#include "pycore_runtime.h" // _PyRuntime +#include "pycore_identifier.h" // _Py_Identifier +#include "pycore_object.h" // PyDictOrValues + +// Unsafe flavor of PyDict_GetItemWithError(): no error checking +extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); + +extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, + int (*predicate)(PyObject *value)); + +// "KnownHash" variants +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, + PyObject *item, Py_hash_t hash); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, + Py_hash_t hash); +extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); + +// "Id" variants +extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, + _Py_Identifier *key); +extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); +extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); +extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); + +extern int _PyDict_Next( + PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); + +extern int _PyDict_HasOnlyStringKeys(PyObject *mp); + +extern void _PyDict_MaybeUntrack(PyObject *mp); + +// Export for '_ctypes' shared extension +PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); + +#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) + +/* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, + the first occurrence of a key wins, if override is 1, the last occurrence + of a key wins, if override is 2, a KeyError with conflicting key as + argument is raised. +*/ +extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); + +extern void _PyDict_DebugMallocStats(FILE *out); + + +/* _PyDictView */ + +typedef struct { + PyObject_HEAD + PyDictObject *dv_dict; +} _PyDictViewObject; + +extern PyObject* _PyDictView_New(PyObject *, PyTypeObject *); +extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other); /* runtime lifecycle */ @@ -42,6 +96,8 @@ extern uint32_t _PyDictKeys_GetVersionForCurrentState( extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); +extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); + /* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. */ @@ -55,7 +111,11 @@ extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); -extern PyObject *_PyDict_Pop_KnownHash(PyObject *, PyObject *, Py_hash_t, PyObject *); +extern int _PyDict_Pop_KnownHash( + PyDictObject *dict, + PyObject *key, + Py_hash_t hash, + PyObject **result); #define DKIX_EMPTY (-1) #define DKIX_DUMMY (-2) /* Used internally */ @@ -176,6 +236,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); +extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); extern PyObject *_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index 4d9681d59a64f7..c5cfdf4ce8f823 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -35,6 +35,9 @@ struct _dtoa_state { /* The size of the Bigint freelist */ #define Bigint_Kmax 7 +/* The size of the cached powers of 5 array */ +#define Bigint_Pow5size 8 + #ifndef PRIVATE_MEM #define PRIVATE_MEM 2304 #endif @@ -42,9 +45,10 @@ struct _dtoa_state { ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) struct _dtoa_state { - /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + // p5s is an array of powers of 5 of the form: + // 5**(2**(i+2)) for 0 <= i < Bigint_Pow5size + struct Bigint *p5s[Bigint_Pow5size]; // XXX This should be freed during runtime fini. - struct Bigint *p5s; struct Bigint *freelist[Bigint_Kmax+1]; double preallocated[Bigint_PREALLOC_SIZE]; double *preallocated_next; @@ -57,16 +61,18 @@ struct _dtoa_state { #endif // !Py_USING_MEMORY_DEBUGGER -/* These functions are used by modules compiled as C extension like math: - they must be exported. */ - -PyAPI_FUNC(double) _Py_dg_strtod(const char *str, char **ptr); -PyAPI_FUNC(char *) _Py_dg_dtoa(double d, int mode, int ndigits, - int *decpt, int *sign, char **rve); -PyAPI_FUNC(void) _Py_dg_freedtoa(char *s); +extern double _Py_dg_strtod(const char *str, char **ptr); +extern char* _Py_dg_dtoa(double d, int mode, int ndigits, + int *decpt, int *sign, char **rve); +extern void _Py_dg_freedtoa(char *s); #endif // _PY_SHORT_FLOAT_REPR == 1 + +extern PyStatus _PyDtoa_Init(PyInterpreterState *interp); +extern void _PyDtoa_Fini(PyInterpreterState *interp); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_emscripten_signal.h b/Include/internal/pycore_emscripten_signal.h index 8b3287d85da4b2..754193e21dec5a 100644 --- a/Include/internal/pycore_emscripten_signal.h +++ b/Include/internal/pycore_emscripten_signal.h @@ -3,6 +3,10 @@ #if defined(__EMSCRIPTEN__) +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + void _Py_CheckEmscriptenSignals(void); @@ -14,6 +18,7 @@ _Py_CheckEmscriptenSignalsPeriodically(void); #define _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY() _Py_CheckEmscriptenSignalsPeriodically() extern int Py_EMSCRIPTEN_SIGNAL_HANDLING; +extern int _Py_emscripten_signal_clock; #else diff --git a/Include/internal/pycore_emscripten_trampoline.h b/Include/internal/pycore_emscripten_trampoline.h new file mode 100644 index 00000000000000..e519c99ad86cce --- /dev/null +++ b/Include/internal/pycore_emscripten_trampoline.h @@ -0,0 +1,81 @@ +#ifndef Py_EMSCRIPTEN_TRAMPOLINE_H +#define Py_EMSCRIPTEN_TRAMPOLINE_H + +#include "pycore_runtime.h" // _PyRuntimeState + +/** + * C function call trampolines to mitigate bad function pointer casts. + * + * Section 6.3.2.3, paragraph 8 reads: + * + * A pointer to a function of one type may be converted to a pointer to a + * function of another type and back again; the result shall compare equal to + * the original pointer. If a converted pointer is used to call a function + * whose type is not compatible with the pointed-to type, the behavior is + * undefined. + * + * Typical native ABIs ignore additional arguments or fill in missing values + * with 0/NULL in function pointer cast. Compilers do not show warnings when a + * function pointer is explicitly casted to an incompatible type. + * + * Bad fpcasts are an issue in WebAssembly. WASM's indirect_call has strict + * function signature checks. Argument count, types, and return type must match. + * + * Third party code unintentionally rely on problematic fpcasts. The call + * trampoline mitigates common occurrences of bad fpcasts on Emscripten. + */ + +#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) + +void _Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime); + +PyObject* +_PyEM_TrampolineCall_JavaScript(PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); + +PyObject* +_PyEM_TrampolineCall_Reflection(PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); + +#define _PyEM_TrampolineCall(meth, self, args, kw) \ + ((_PyRuntime.wasm_type_reflection_available) ? \ + (_PyEM_TrampolineCall_Reflection((PyCFunctionWithKeywords)(meth), (self), (args), (kw))) : \ + (_PyEM_TrampolineCall_JavaScript((PyCFunctionWithKeywords)(meth), (self), (args), (kw)))) + +#define _PyCFunction_TrampolineCall(meth, self, args) \ + _PyEM_TrampolineCall( \ + (*(PyCFunctionWithKeywords)(void(*)(void))(meth)), (self), (args), NULL) + +#define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \ + _PyEM_TrampolineCall((meth), (self), (args), (kw)) + +#define descr_set_trampoline_call(set, obj, value, closure) \ + ((int)_PyEM_TrampolineCall((PyCFunctionWithKeywords)(set), (obj), (value), (PyObject*)(closure))) + +#define descr_get_trampoline_call(get, obj, closure) \ + _PyEM_TrampolineCall((PyCFunctionWithKeywords)(get), (obj), (PyObject*)(closure), NULL) + + +#else // defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) + +#define _Py_EmscriptenTrampoline_Init(runtime) + +#define _PyCFunction_TrampolineCall(meth, self, args) \ + (meth)((self), (args)) + +#define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \ + (meth)((self), (args), (kw)) + +#define descr_set_trampoline_call(set, obj, value, closure) \ + (set)((obj), (value), (closure)) + +#define descr_get_trampoline_call(get, obj, closure) \ + (get)((obj), (closure)) + +#endif // defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) + +#endif // ndef Py_EMSCRIPTEN_SIGNAL_H diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h index e6aec7745a6479..6dd7d8d7ca9792 100644 --- a/Include/internal/pycore_faulthandler.h +++ b/Include/internal/pycore_faulthandler.h @@ -9,7 +9,7 @@ extern "C" { #endif #ifdef HAVE_SIGACTION -# include +# include // sigaction #endif diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index ef6642d00f1b54..5c55282fa39e6f 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -5,12 +5,20 @@ extern "C" { #endif #ifndef Py_BUILD_CORE -# error "Py_BUILD_CORE must be defined to include this header" +# error "this header requires Py_BUILD_CORE define" #endif -#include /* struct lconv */ +#include // struct lconv +/* A routine to check if a file descriptor can be select()-ed. */ +#ifdef _MSC_VER + /* On Windows, any socket fd can be select()-ed, no matter how high */ + #define _PyIsSelectable_fd(FD) (1) +#else + #define _PyIsSelectable_fd(FD) ((unsigned int)(FD) < (unsigned int)FD_SETSIZE) +#endif + struct _fileutils_state { int force_ascii; }; @@ -27,8 +35,10 @@ typedef enum { _Py_ERROR_OTHER } _Py_error_handler; +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _Py_DecodeLocaleEx( const char *arg, wchar_t **wstr, @@ -37,6 +47,7 @@ PyAPI_FUNC(int) _Py_DecodeLocaleEx( int current_locale, _Py_error_handler errors); +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _Py_EncodeLocaleEx( const wchar_t *text, char **str, @@ -45,11 +56,11 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( int current_locale, _Py_error_handler errors); -PyAPI_FUNC(char*) _Py_EncodeLocaleRaw( +extern char* _Py_EncodeLocaleRaw( const wchar_t *text, size_t *error_pos); -PyAPI_FUNC(PyObject *) _Py_device_encoding(int); +extern PyObject* _Py_device_encoding(int); #if defined(MS_WINDOWS) || defined(__APPLE__) /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). @@ -90,47 +101,54 @@ struct _Py_stat_struct { # define _Py_stat_struct stat #endif +// Export for 'mmap' shared extension PyAPI_FUNC(int) _Py_fstat( int fd, struct _Py_stat_struct *status); +// Export for 'mmap' shared extension PyAPI_FUNC(int) _Py_fstat_noraise( int fd, struct _Py_stat_struct *status); +// Export for '_tkinter' shared extension PyAPI_FUNC(int) _Py_stat( PyObject *path, struct stat *status); +// Export for 'select' shared extension (Solaris newDevPollObject()) PyAPI_FUNC(int) _Py_open( const char *pathname, int flags); +// Export for '_posixsubprocess' shared extension PyAPI_FUNC(int) _Py_open_noraise( const char *pathname, int flags); -PyAPI_FUNC(FILE *) _Py_wfopen( +extern FILE* _Py_wfopen( const wchar_t *path, const wchar_t *mode); -PyAPI_FUNC(Py_ssize_t) _Py_read( +extern Py_ssize_t _Py_read( int fd, void *buf, size_t count); +// Export for 'select' shared extension (Solaris devpoll_flush()) PyAPI_FUNC(Py_ssize_t) _Py_write( int fd, const void *buf, size_t count); +// Export for '_posixsubprocess' shared extension PyAPI_FUNC(Py_ssize_t) _Py_write_noraise( int fd, const void *buf, size_t count); #ifdef HAVE_READLINK -PyAPI_FUNC(int) _Py_wreadlink( +extern int _Py_wreadlink( const wchar_t *path, wchar_t *buf, /* Number of characters of 'buf' buffer @@ -139,7 +157,7 @@ PyAPI_FUNC(int) _Py_wreadlink( #endif #ifdef HAVE_REALPATH -PyAPI_FUNC(wchar_t*) _Py_wrealpath( +extern wchar_t* _Py_wrealpath( const wchar_t *path, wchar_t *resolved_path, /* Number of characters of 'resolved_path' buffer @@ -147,34 +165,38 @@ PyAPI_FUNC(wchar_t*) _Py_wrealpath( size_t resolved_path_len); #endif -PyAPI_FUNC(wchar_t*) _Py_wgetcwd( +extern wchar_t* _Py_wgetcwd( wchar_t *buf, /* Number of characters of 'buf' buffer including the trailing NUL character */ size_t buflen); -PyAPI_FUNC(int) _Py_get_inheritable(int fd); +extern int _Py_get_inheritable(int fd); +// Export for '_socket' shared extension PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable, int *atomic_flag_works); +// Export for '_posixsubprocess' shared extension PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, int *atomic_flag_works); +// Export for '_socket' shared extension PyAPI_FUNC(int) _Py_dup(int fd); -PyAPI_FUNC(int) _Py_get_blocking(int fd); +extern int _Py_get_blocking(int fd); -PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); +extern int _Py_set_blocking(int fd, int blocking); #ifdef MS_WINDOWS -PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd); +extern void* _Py_get_osfhandle_noraise(int fd); +// Export for '_testconsole' shared extension PyAPI_FUNC(void*) _Py_get_osfhandle(int fd); -PyAPI_FUNC(int) _Py_open_osfhandle_noraise(void *handle, int flags); +extern int _Py_open_osfhandle_noraise(void *handle, int flags); -PyAPI_FUNC(int) _Py_open_osfhandle(void *handle, int flags); +extern int _Py_open_osfhandle(void *handle, int flags); #endif /* MS_WINDOWS */ // This is used after getting NULL back from Py_DecodeLocale(). @@ -183,9 +205,9 @@ PyAPI_FUNC(int) _Py_open_osfhandle(void *handle, int flags); ? _PyStatus_ERR("cannot decode " NAME) \ : _PyStatus_NO_MEMORY() -PyAPI_DATA(int) _Py_HasFileSystemDefaultEncodeErrors; +extern int _Py_HasFileSystemDefaultEncodeErrors; -PyAPI_FUNC(int) _Py_DecodeUTF8Ex( +extern int _Py_DecodeUTF8Ex( const char *arg, Py_ssize_t arglen, wchar_t **wstr, @@ -193,7 +215,7 @@ PyAPI_FUNC(int) _Py_DecodeUTF8Ex( const char **reason, _Py_error_handler errors); -PyAPI_FUNC(int) _Py_EncodeUTF8Ex( +extern int _Py_EncodeUTF8Ex( const wchar_t *text, char **str, size_t *error_pos, @@ -201,7 +223,7 @@ PyAPI_FUNC(int) _Py_EncodeUTF8Ex( int raw_malloc, _Py_error_handler errors); -PyAPI_FUNC(wchar_t*) _Py_DecodeUTF8_surrogateescape( +extern wchar_t* _Py_DecodeUTF8_surrogateescape( const char *arg, Py_ssize_t arglen, size_t *wlen); @@ -209,25 +231,26 @@ PyAPI_FUNC(wchar_t*) _Py_DecodeUTF8_surrogateescape( extern int _Py_wstat(const wchar_t *, struct stat *); -PyAPI_FUNC(int) _Py_GetForceASCII(void); +extern int _Py_GetForceASCII(void); /* Reset "force ASCII" mode (if it was initialized). This function should be called when Python changes the LC_CTYPE locale, so the "force ASCII" mode can be detected again on the new locale encoding. */ -PyAPI_FUNC(void) _Py_ResetForceASCII(void); +extern void _Py_ResetForceASCII(void); -PyAPI_FUNC(int) _Py_GetLocaleconvNumeric( +extern int _Py_GetLocaleconvNumeric( struct lconv *lc, PyObject **decimal_point, PyObject **thousands_sep); +// Export for '_posixsubprocess' (on macOS) PyAPI_FUNC(void) _Py_closerange(int first, int last); -PyAPI_FUNC(wchar_t*) _Py_GetLocaleEncoding(void); -PyAPI_FUNC(PyObject*) _Py_GetLocaleEncodingObject(void); +extern wchar_t* _Py_GetLocaleEncoding(void); +extern PyObject* _Py_GetLocaleEncodingObject(void); #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION extern int _Py_LocaleUsesNonUnicodeWchar(void); @@ -246,19 +269,23 @@ extern int _Py_abspath(const wchar_t *path, wchar_t **abspath_p); #ifdef MS_WINDOWS extern int _PyOS_getfullpathname(const wchar_t *path, wchar_t **abspath_p); #endif -extern wchar_t * _Py_join_relfile(const wchar_t *dirname, - const wchar_t *relfile); +extern wchar_t* _Py_join_relfile(const wchar_t *dirname, + const wchar_t *relfile); extern int _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize); extern size_t _Py_find_basename(const wchar_t *filename); -PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); + +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); + +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *length); // The Windows Games API family does not provide these functions // so provide our own implementations. Remove them in case they get added // to the Games API family #if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) -#include +#include // HRESULT extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ @@ -285,6 +312,18 @@ extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootE # define _Py_END_SUPPRESS_IPH #endif /* _MSC_VER >= 1900 */ +// Export for 'select' shared extension (Argument Clinic code) +PyAPI_FUNC(int) _PyLong_FileDescriptor_Converter(PyObject *, void *); + +// Export for test_peg_generator +PyAPI_FUNC(char*) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*); + +extern int _PyFile_Flush(PyObject *); + +#ifndef MS_WINDOWS +extern int _Py_GetTicksPerSecond(long *ticks_per_second); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_fileutils_windows.h b/Include/internal/pycore_fileutils_windows.h index e804d385e76708..b79aa9fb465376 100644 --- a/Include/internal/pycore_fileutils_windows.h +++ b/Include/internal/pycore_fileutils_windows.h @@ -5,7 +5,7 @@ extern "C" { #endif #ifndef Py_BUILD_CORE -# error "Py_BUILD_CORE must be defined to include this header" +# error "this header requires Py_BUILD_CORE define" #endif #ifdef MS_WINDOWS diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 27c63bc87f3ee3..4e5474841bc25d 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -9,6 +9,8 @@ extern "C" { #endif +#include "pycore_unicodeobject.h" // _PyUnicodeWriter + /* runtime lifecycle */ extern void _PyFloat_InitState(PyInterpreterState *); @@ -53,18 +55,25 @@ struct _Py_float_state { void _PyFloat_ExactDealloc(PyObject *op); -PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); +extern void _PyFloat_DebugMallocStats(FILE* out); /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( +extern int _PyFloat_FormatAdvancedWriter( _PyUnicodeWriter *writer, PyObject *obj, PyObject *format_spec, Py_ssize_t start, Py_ssize_t end); +extern PyObject* _Py_string_to_number_with_underscores( + const char *str, Py_ssize_t len, const char *what, PyObject *obj, void *arg, + PyObject *(*innerfunc)(const char *, Py_ssize_t, void *)); + +extern double _Py_parse_inf_or_nan(const char *p, char **endptr); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 720feb18636959..58fed46886ea45 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -11,103 +11,26 @@ extern "C" { #include "pycore_opcode_utils.h" #include "pycore_compile.h" - -typedef struct { - int i_opcode; - int i_oparg; - _PyCompilerSrcLocation i_loc; - struct _PyCfgBasicblock_ *i_target; /* target block (if jump instruction) */ - struct _PyCfgBasicblock_ *i_except; /* target block when exception is raised */ -} _PyCfgInstruction; - typedef struct { int id; } _PyCfgJumpTargetLabel; +struct _PyCfgBuilder; -typedef struct { - struct _PyCfgBasicblock_ *handlers[CO_MAXBLOCKS+1]; - int depth; -} _PyCfgExceptStack; - -typedef struct _PyCfgBasicblock_ { - /* Each basicblock in a compilation unit is linked via b_list in the - reverse order that the block are allocated. b_list points to the next - block in this list, not to be confused with b_next, which is next by - control flow. */ - struct _PyCfgBasicblock_ *b_list; - /* The label of this block if it is a jump target, -1 otherwise */ - _PyCfgJumpTargetLabel b_label; - /* Exception stack at start of block, used by assembler to create the exception handling table */ - _PyCfgExceptStack *b_exceptstack; - /* pointer to an array of instructions, initially NULL */ - _PyCfgInstruction *b_instr; - /* If b_next is non-NULL, it is a pointer to the next - block reached by normal control flow. */ - struct _PyCfgBasicblock_ *b_next; - /* number of instructions used */ - int b_iused; - /* length of instruction array (b_instr) */ - int b_ialloc; - /* Used by add_checks_for_loads_of_unknown_variables */ - uint64_t b_unsafe_locals_mask; - /* Number of predecessors that a block has. */ - int b_predecessors; - /* depth of stack upon entry of block, computed by stackdepth() */ - int b_startdepth; - /* instruction offset for block, computed by assemble_jump_offsets() */ - int b_offset; - /* Basic block is an exception handler that preserves lasti */ - unsigned b_preserve_lasti : 1; - /* Used by compiler passes to mark whether they have visited a basic block. */ - unsigned b_visited : 1; - /* b_except_handler is used by the cold-detection algorithm to mark exception targets */ - unsigned b_except_handler : 1; - /* b_cold is true if this block is not perf critical (like an exception handler) */ - unsigned b_cold : 1; - /* b_warm is used by the cold-detection algorithm to mark blocks which are definitely not cold */ - unsigned b_warm : 1; -} _PyCfgBasicblock; - -int _PyBasicblock_InsertInstruction(_PyCfgBasicblock *block, int pos, _PyCfgInstruction *instr); +int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); +int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); -typedef struct cfg_builder_ { - /* The entryblock, at which control flow begins. All blocks of the - CFG are reachable through the b_next links */ - _PyCfgBasicblock *g_entryblock; - /* Pointer to the most recently allocated block. By following - b_list links, you can reach all allocated blocks. */ - _PyCfgBasicblock *g_block_list; - /* pointer to the block currently being constructed */ - _PyCfgBasicblock *g_curblock; - /* label for the next instruction to be placed */ - _PyCfgJumpTargetLabel g_current_label; -} _PyCfgBuilder; +struct _PyCfgBuilder* _PyCfgBuilder_New(void); +void _PyCfgBuilder_Free(struct _PyCfgBuilder *g); +int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g); -int _PyCfgBuilder_UseLabel(_PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); -int _PyCfgBuilder_Addop(_PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); - -int _PyCfgBuilder_Init(_PyCfgBuilder *g); -void _PyCfgBuilder_Fini(_PyCfgBuilder *g); - -_PyCfgInstruction* _PyCfg_BasicblockLastInstr(const _PyCfgBasicblock *b); -int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, - int code_flags, int nlocals, int nparams, int firstlineno); -int _PyCfg_Stackdepth(_PyCfgBasicblock *entryblock, int code_flags); -void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock); -int _PyCfg_ResolveJumps(_PyCfgBuilder *g); - - -static inline int -basicblock_nofallthrough(const _PyCfgBasicblock *b) { - _PyCfgInstruction *last = _PyCfg_BasicblockLastInstr(b); - return (last && - (IS_SCOPE_EXIT_OPCODE(last->i_opcode) || - IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode))); -} +int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, + int nlocals, int nparams, int firstlineno); -#define BB_NO_FALLTHROUGH(B) (basicblock_nofallthrough(B)) -#define BB_HAS_FALLTHROUGH(B) (!basicblock_nofallthrough(B)) +int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_InstructionSequence *seq); +int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, + int code_flags, int *stackdepth, int *nlocalsplus, + _PyCompile_InstructionSequence *seq); PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index a72e03f1438fc8..0f9e7333cf1e1c 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -4,9 +4,13 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #include -#include -#include "pycore_code.h" // STATS +#include // offsetof() +#include "pycore_code.h" // STATS /* See Objects/frame_layout.md for an explanation of the frame stack * including explanation of the PyFrameObject and _PyInterpreterFrame @@ -32,13 +36,17 @@ extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code); /* other API */ typedef enum _framestate { - FRAME_CREATED = -2, - FRAME_SUSPENDED = -1, + FRAME_CREATED = -3, + FRAME_SUSPENDED = -2, + FRAME_SUSPENDED_YIELD_FROM = -1, FRAME_EXECUTING = 0, FRAME_COMPLETED = 1, FRAME_CLEARED = 4 } PyFrameState; +#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM) +#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED) + enum _frameowner { FRAME_OWNED_BY_THREAD = 0, FRAME_OWNED_BY_GENERATOR = 1, @@ -47,46 +55,41 @@ enum _frameowner { }; typedef struct _PyInterpreterFrame { - PyCodeObject *f_code; /* Strong reference */ + PyObject *f_executable; /* Strong reference */ struct _PyInterpreterFrame *previous; PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ - // NOTE: This is not necessarily the last instruction started in the given - // frame. Rather, it is the code unit *prior to* the *next* instruction. For - // example, it may be an inline CACHE entry, an instruction we just jumped - // over, or (in the case of a newly-created frame) a totally invalid value: - _Py_CODEUNIT *prev_instr; + _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ int stacktop; /* Offset of TOS from localsplus */ - /* The return_offset determines where a `RETURN` should go in the caller, - * relative to `prev_instr`. - * It is only meaningful to the callee, - * so it needs to be set in any CALL (to a Python function) - * or SEND (to a coroutine or generator). - * If there is no callee, then it is meaningless. */ - uint16_t return_offset; + uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ PyObject *localsplus[1]; } _PyInterpreterFrame; #define _PyInterpreterFrame_LASTI(IF) \ - ((int)((IF)->prev_instr - _PyCode_CODE((IF)->f_code))) + ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF)))) + +static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { + assert(PyCode_Check(f->f_executable)); + return (PyCodeObject *)f->f_executable; +} static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) { - return f->localsplus + f->f_code->co_nlocalsplus; + return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus; } static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) { - assert(f->stacktop > f->f_code->co_nlocalsplus); + assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); assert(f->localsplus[f->stacktop-1] != NULL); return f->localsplus[f->stacktop-1]; } static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) { - assert(f->stacktop > f->f_code->co_nlocalsplus); + assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); f->stacktop--; return f->localsplus[f->stacktop]; } @@ -119,13 +122,13 @@ _PyFrame_Initialize( PyObject *locals, PyCodeObject *code, int null_locals_from) { frame->f_funcobj = (PyObject *)func; - frame->f_code = (PyCodeObject *)Py_NewRef(code); + frame->f_executable = Py_NewRef(code); frame->f_builtins = func->func_builtins; frame->f_globals = func->func_globals; frame->f_locals = locals; frame->stacktop = code->co_nlocalsplus; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code) - 1; + frame->instr_ptr = _PyCode_CODE(code); frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; @@ -172,8 +175,11 @@ _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) static inline bool _PyFrame_IsIncomplete(_PyInterpreterFrame *frame) { + if (frame->owner == FRAME_OWNED_BY_CSTACK) { + return true; + } return frame->owner != FRAME_OWNED_BY_GENERATOR && - frame->prev_instr < _PyCode_CODE(frame->f_code) + frame->f_code->_co_firsttraceable; + frame->instr_ptr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; } static inline _PyInterpreterFrame * @@ -188,7 +194,7 @@ _PyFrame_GetFirstComplete(_PyInterpreterFrame *frame) static inline _PyInterpreterFrame * _PyThreadState_GetFrame(PyThreadState *tstate) { - return _PyFrame_GetFirstComplete(tstate->cframe->current_frame); + return _PyFrame_GetFirstComplete(tstate->current_frame); } /* For use by _PyFrame_GetFrameObject @@ -226,6 +232,9 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); + int _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); @@ -264,6 +273,30 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_l return new_frame; } +/* Pushes a trampoline frame without checking for space. + * Must be guarded by _PyThreadState_HasStackSpace() */ +static inline _PyInterpreterFrame * +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth) +{ + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; + tstate->datastack_top += code->co_framesize; + assert(tstate->datastack_top < tstate->datastack_limit); + frame->f_funcobj = Py_None; + frame->f_executable = Py_NewRef(code); +#ifdef Py_DEBUG + frame->f_builtins = NULL; + frame->f_globals = NULL; +#endif + frame->f_locals = NULL; + frame->stacktop = code->co_nlocalsplus + stackdepth; + frame->frame_obj = NULL; + frame->instr_ptr = _PyCode_CODE(code); + frame->owner = FRAME_OWNED_BY_THREAD; + frame->return_offset = 0; + return frame; +} + static inline PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) { diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 11988149843fef..3f3da8a44b77e4 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -8,15 +8,32 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +extern PyObject* _PyFunction_Vectorcall( + PyObject *func, + PyObject *const *stack, + size_t nargsf, + PyObject *kwnames); + #define FUNC_MAX_WATCHERS 8 +#define FUNC_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ struct _py_func_state { uint32_t next_version; + // Borrowed references to function objects whose + // func_version % FUNC_VERSION_CACHE_SIZE + // once was equal to the index in the table. + // They are cleared when the function is deallocated. + PyFunctionObject *func_version_cache[FUNC_VERSION_CACHE_SIZE]; }; extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); +extern void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); +PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); + +extern PyObject *_Py_set_function_type_params( + PyThreadState* unused, PyObject *func, PyObject *type_params); #ifdef __cplusplus } diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index b3abe2030a03da..2d33aa76d78229 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -19,10 +19,21 @@ typedef struct { uintptr_t _gc_prev; } PyGC_Head; +#define _PyGC_Head_UNUSED PyGC_Head + + +/* Get an object's GC head */ static inline PyGC_Head* _Py_AS_GC(PyObject *op) { - return (_Py_CAST(PyGC_Head*, op) - 1); + char *gc = ((char*)op) - sizeof(PyGC_Head); + return (PyGC_Head*)gc; } -#define _PyGC_Head_UNUSED PyGC_Head + +/* Get the object given the GC head */ +static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { + char *op = ((char *)gc) + sizeof(PyGC_Head); + return (PyObject *)op; +} + /* True if the object is currently tracked by the GC. */ static inline int _PyObject_GC_IS_TRACKED(PyObject *op) { @@ -57,19 +68,19 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { uintptr_t next = gc->_gc_next; - return _Py_CAST(PyGC_Head*, next); + return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - gc->_gc_next = _Py_CAST(uintptr_t, next); + gc->_gc_next = (uintptr_t)next; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. static inline PyGC_Head* _PyGCHead_PREV(PyGC_Head *gc) { uintptr_t prev = (gc->_gc_prev & _PyGC_PREV_MASK); - return _Py_CAST(PyGC_Head*, prev); + return (PyGC_Head*)prev; } static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { - uintptr_t uprev = _Py_CAST(uintptr_t, prev); + uintptr_t uprev = (uintptr_t)prev; assert((uprev & ~_PyGC_PREV_MASK) == 0); gc->_gc_prev = ((gc->_gc_prev & ~_PyGC_PREV_MASK) | uprev); } diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index dc60b4ca705112..cf58a2750a31f9 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -9,9 +9,21 @@ extern "C" { #endif extern PyObject *_PyGen_yf(PyGenObject *); +extern void _PyGen_Finalize(PyObject *self); + +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); + +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); + extern PyObject *_PyCoro_GetAwaitableIter(PyObject *o); extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +extern PyTypeObject _PyCoroWrapper_Type; +extern PyTypeObject _PyAsyncGenWrappedValue_Type; +extern PyTypeObject _PyAsyncGenAThrow_Type; + /* runtime lifecycle */ extern void _PyAsyncGen_Fini(PyInterpreterState *); diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index 8ebad37b686cd4..19b0d23a68568a 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -8,8 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" /* _Py_atomic_address */ -#include "pycore_condvar.h" /* PyCOND_T */ +#include "pycore_condvar.h" // PyCOND_T #ifndef Py_HAVE_CONDVAR # error You need either a POSIX-compatible or a Windows system! @@ -25,10 +24,10 @@ struct _gil_runtime_state { unsigned long interval; /* Last PyThreadState holding / having held the GIL. This helps us know whether anyone else was scheduled after we dropped the GIL. */ - _Py_atomic_address last_holder; + PyThreadState* last_holder; /* Whether the GIL is already taken (-1 if uninitialized). This is atomic because it can be read without any lock taken in ceval.c. */ - _Py_atomic_int locked; + int locked; /* Number of GIL switches since the beginning. */ unsigned long switch_number; /* This condition variable allows one or several threads to wait diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 64d9384df9c5c5..327fcc24cb29f1 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -8,10 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_gc.h" // PyGC_Head +#include "pycore_context.h" // _PyContextTokenMissing +#include "pycore_gc.h" // _PyGC_Head_UNUSED #include "pycore_global_strings.h" // struct _Py_global_strings #include "pycore_hamt.h" // PyHamtNode_Bitmap -#include "pycore_context.h" // _PyContextTokenMissing +#include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_typeobject.h" // pytype_slotdef @@ -28,6 +29,11 @@ extern "C" { #define _Py_SINGLETON(NAME) \ _Py_GLOBAL_OBJECT(singletons.NAME) +struct _Py_cached_objects { + // XXX We could statically allocate the hashtable. + _Py_hashtable_t *interned_strings; +}; + struct _Py_static_objects { struct { /* Small integers are preallocated in this array so that they @@ -68,6 +74,13 @@ struct _Py_interp_cached_objects { PyObject *type_slots_pname; pytype_slotdef *type_slots_ptrs[MAX_EQUIV]; + /* TypeVar and related types */ + PyTypeObject *generic_type; + PyTypeObject *typevar_type; + PyTypeObject *typevartuple_type; + PyTypeObject *paramspec_type; + PyTypeObject *paramspecargs_type; + PyTypeObject *paramspeckwargs_type; }; #define _Py_INTERP_STATIC_OBJECT(interp, NAME) \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 7e495817981f06..89ec8cbbbcd649 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -12,9 +12,8 @@ extern "C" { static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (Py_REFCNT(obj) < _Py_IMMORTAL_REFCNT) { - _PyObject_ASSERT_FAILED_MSG(obj, - "immortal object has less refcnt than expected " - "_Py_IMMORTAL_REFCNT"); + fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); + _PyObject_Dump(obj); } } #endif @@ -548,6 +547,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_listcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_module)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_null)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_setcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_string)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_unknown)); @@ -555,15 +555,18 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_close_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_open_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_percent)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(defaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(shim_name)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED)); @@ -602,6 +605,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__class__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__class_getitem__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__classcell__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__classdict__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__classdictcell__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__complex__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__contains__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__copy__)); @@ -664,6 +669,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lshift__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lt__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__main__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__match_args__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__matmul__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__missing__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__mod__)); @@ -724,6 +730,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasshook__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__truediv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__trunc__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__type_params__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_is_unpacked_typevartuple__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_prepare_subst__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_subst__)); @@ -774,6 +781,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(a)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(abs_tol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(access)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aclose)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add_done_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); @@ -784,7 +792,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(asend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ast)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(athrow)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(authorizer_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(autocommit)); @@ -795,6 +805,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(big)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(binary_form)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(block)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer_size)); @@ -815,6 +826,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_exception_handler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_soon)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); @@ -850,11 +862,13 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(contravariant)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cookie)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(copy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(copyreg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(coro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(count)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(covariant)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cwd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); @@ -883,7 +897,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dont_inherit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(duration)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(e)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); @@ -899,12 +912,13 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(f)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); @@ -928,7 +942,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(frequency)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); @@ -936,6 +949,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(g)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); @@ -949,6 +963,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groupindex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hash_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(header)); @@ -957,6 +972,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(imag)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); @@ -964,6 +980,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(incoming)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inf)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(infer_variance)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inheritable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_bytes)); @@ -976,6 +993,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(instructions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intern)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intersection)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(interval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_running)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isatty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isinstance)); @@ -1053,10 +1071,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(narg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_limit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newlines)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(next)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nlocals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(node_depth)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(node_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ns)); @@ -1080,6 +1100,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(optimize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(options)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(order)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); @@ -1105,6 +1126,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_handler)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_routine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(proto)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(protocol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps1)); @@ -1164,7 +1186,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sleep)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sock)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sort)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source_traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src)); @@ -1177,6 +1198,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdout)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(step)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(steps)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(store_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strategy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strftime)); @@ -1205,7 +1227,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); @@ -1213,6 +1234,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(twice)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(txt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tz)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8ebfee85c87c23..62c3ee3ae2a0bd 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -33,6 +33,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(anon_lambda, "") STRUCT_FOR_STR(anon_listcomp, "") STRUCT_FOR_STR(anon_module, "") + STRUCT_FOR_STR(anon_null, "") STRUCT_FOR_STR(anon_setcomp, "") STRUCT_FOR_STR(anon_string, "") STRUCT_FOR_STR(anon_unknown, "") @@ -40,15 +41,18 @@ struct _Py_global_strings { STRUCT_FOR_STR(dbl_close_br, "}}") STRUCT_FOR_STR(dbl_open_br, "{{") STRUCT_FOR_STR(dbl_percent, "%%") + STRUCT_FOR_STR(defaults, ".defaults") STRUCT_FOR_STR(dot, ".") STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") + STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") + STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") STRUCT_FOR_STR(newline, "\n") STRUCT_FOR_STR(open_br, "{") STRUCT_FOR_STR(percent, "%") - STRUCT_FOR_STR(shim_name, "") + STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; @@ -90,6 +94,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(__class__) STRUCT_FOR_ID(__class_getitem__) STRUCT_FOR_ID(__classcell__) + STRUCT_FOR_ID(__classdict__) + STRUCT_FOR_ID(__classdictcell__) STRUCT_FOR_ID(__complex__) STRUCT_FOR_ID(__contains__) STRUCT_FOR_ID(__copy__) @@ -152,6 +158,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__lshift__) STRUCT_FOR_ID(__lt__) STRUCT_FOR_ID(__main__) + STRUCT_FOR_ID(__match_args__) STRUCT_FOR_ID(__matmul__) STRUCT_FOR_ID(__missing__) STRUCT_FOR_ID(__mod__) @@ -212,6 +219,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__subclasshook__) STRUCT_FOR_ID(__truediv__) STRUCT_FOR_ID(__trunc__) + STRUCT_FOR_ID(__type_params__) STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__) STRUCT_FOR_ID(__typing_prepare_subst__) STRUCT_FOR_ID(__typing_subst__) @@ -262,6 +270,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(a) STRUCT_FOR_ID(abs_tol) STRUCT_FOR_ID(access) + STRUCT_FOR_ID(aclose) STRUCT_FOR_ID(add) STRUCT_FOR_ID(add_done_callback) STRUCT_FOR_ID(after_in_child) @@ -272,7 +281,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) + STRUCT_FOR_ID(asend) STRUCT_FOR_ID(ast) + STRUCT_FOR_ID(athrow) STRUCT_FOR_ID(attribute) STRUCT_FOR_ID(authorizer_callback) STRUCT_FOR_ID(autocommit) @@ -283,6 +294,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(big) STRUCT_FOR_ID(binary_form) STRUCT_FOR_ID(block) + STRUCT_FOR_ID(bound) STRUCT_FOR_ID(buffer) STRUCT_FOR_ID(buffer_callback) STRUCT_FOR_ID(buffer_size) @@ -303,6 +315,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(call) STRUCT_FOR_ID(call_exception_handler) STRUCT_FOR_ID(call_soon) + STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) STRUCT_FOR_ID(capath) STRUCT_FOR_ID(category) @@ -338,11 +351,13 @@ struct _Py_global_strings { STRUCT_FOR_ID(compile_mode) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) + STRUCT_FOR_ID(contravariant) STRUCT_FOR_ID(cookie) STRUCT_FOR_ID(copy) STRUCT_FOR_ID(copyreg) STRUCT_FOR_ID(coro) STRUCT_FOR_ID(count) + STRUCT_FOR_ID(covariant) STRUCT_FOR_ID(cwd) STRUCT_FOR_ID(d) STRUCT_FOR_ID(data) @@ -371,7 +386,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(dont_inherit) STRUCT_FOR_ID(dst) STRUCT_FOR_ID(dst_dir_fd) - STRUCT_FOR_ID(duration) STRUCT_FOR_ID(e) STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) @@ -387,12 +401,13 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) - STRUCT_FOR_ID(exc_type) - STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) + STRUCT_FOR_ID(existing_file_name) STRUCT_FOR_ID(exp) STRUCT_FOR_ID(extend) + STRUCT_FOR_ID(extra_tokens) + STRUCT_FOR_ID(f) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) STRUCT_FOR_ID(false) @@ -416,7 +431,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(flush) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) - STRUCT_FOR_ID(frequency) STRUCT_FOR_ID(from_param) STRUCT_FOR_ID(fromlist) STRUCT_FOR_ID(fromtimestamp) @@ -424,6 +438,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fset) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(g) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(genexpr) STRUCT_FOR_ID(get) @@ -437,6 +452,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(globals) STRUCT_FOR_ID(groupindex) STRUCT_FOR_ID(groups) + STRUCT_FOR_ID(h) STRUCT_FOR_ID(handle) STRUCT_FOR_ID(hash_name) STRUCT_FOR_ID(header) @@ -445,6 +461,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(hook) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) + STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) STRUCT_FOR_ID(imag) STRUCT_FOR_ID(importlib) @@ -452,6 +469,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(incoming) STRUCT_FOR_ID(indexgroup) STRUCT_FOR_ID(inf) + STRUCT_FOR_ID(infer_variance) STRUCT_FOR_ID(inheritable) STRUCT_FOR_ID(initial) STRUCT_FOR_ID(initial_bytes) @@ -464,6 +482,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(instructions) STRUCT_FOR_ID(intern) STRUCT_FOR_ID(intersection) + STRUCT_FOR_ID(interval) STRUCT_FOR_ID(is_running) STRUCT_FOR_ID(isatty) STRUCT_FOR_ID(isinstance) @@ -541,10 +560,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(namespaces) STRUCT_FOR_ID(narg) STRUCT_FOR_ID(ndigits) + STRUCT_FOR_ID(new_file_name) STRUCT_FOR_ID(new_limit) STRUCT_FOR_ID(newline) STRUCT_FOR_ID(newlines) STRUCT_FOR_ID(next) + STRUCT_FOR_ID(nlocals) STRUCT_FOR_ID(node_depth) STRUCT_FOR_ID(node_offset) STRUCT_FOR_ID(ns) @@ -568,6 +589,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(optimize) STRUCT_FOR_ID(options) STRUCT_FOR_ID(order) + STRUCT_FOR_ID(origin) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(overlapped) @@ -593,6 +615,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) STRUCT_FOR_ID(progress_handler) + STRUCT_FOR_ID(progress_routine) STRUCT_FOR_ID(proto) STRUCT_FOR_ID(protocol) STRUCT_FOR_ID(ps1) @@ -652,7 +675,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(sleep) STRUCT_FOR_ID(sock) STRUCT_FOR_ID(sort) - STRUCT_FOR_ID(sound) STRUCT_FOR_ID(source) STRUCT_FOR_ID(source_traceback) STRUCT_FOR_ID(src) @@ -665,6 +687,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(stdin) STRUCT_FOR_ID(stdout) STRUCT_FOR_ID(step) + STRUCT_FOR_ID(steps) STRUCT_FOR_ID(store_name) STRUCT_FOR_ID(strategy) STRUCT_FOR_ID(strftime) @@ -693,7 +716,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) - STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) STRUCT_FOR_ID(true) @@ -701,6 +723,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(twice) STRUCT_FOR_ID(txt) STRUCT_FOR_ID(type) + STRUCT_FOR_ID(type_params) STRUCT_FOR_ID(tz) STRUCT_FOR_ID(tzname) STRUCT_FOR_ID(uid) diff --git a/Include/internal/pycore_hashtable.h b/Include/internal/pycore_hashtable.h index 6501ab14d27684..369d49c42bbfcc 100644 --- a/Include/internal/pycore_hashtable.h +++ b/Include/internal/pycore_hashtable.h @@ -70,6 +70,11 @@ struct _Py_hashtable_t { _Py_hashtable_allocator_t alloc; }; +// Export _Py_hashtable functions for '_testinternalcapi' shared extension +PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( + _Py_hashtable_hash_func hash_func, + _Py_hashtable_compare_func compare_func); + /* Hash a pointer (void*) */ PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); @@ -78,10 +83,6 @@ PyAPI_FUNC(int) _Py_hashtable_compare_direct( const void *key1, const void *key2); -PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( - _Py_hashtable_hash_func hash_func, - _Py_hashtable_compare_func compare_func); - PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, @@ -106,6 +107,7 @@ PyAPI_FUNC(int) _Py_hashtable_foreach( void *user_data); PyAPI_FUNC(size_t) _Py_hashtable_size(const _Py_hashtable_t *ht); +PyAPI_FUNC(size_t) _Py_hashtable_len(const _Py_hashtable_t *ht); /* Add a new entry to the hash. The key must not be present in the hash table. Return 0 on success, -1 on memory error. */ diff --git a/Include/internal/pycore_identifier.h b/Include/internal/pycore_identifier.h new file mode 100644 index 00000000000000..cda28810a48196 --- /dev/null +++ b/Include/internal/pycore_identifier.h @@ -0,0 +1,20 @@ +/* String Literals: _Py_Identifier API */ + +#ifndef Py_INTERNAL_IDENTIFIER_H +#define Py_INTERNAL_IDENTIFIER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern PyObject* _PyType_LookupId(PyTypeObject *, _Py_Identifier *); +extern PyObject* _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); +extern int _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_IDENTIFIER_H diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 0a9f24efbdb908..c84f87a831bf38 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -5,6 +5,38 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_lock.h" // PyMutex +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_time.h" // _PyTime_t + +extern int _PyImport_IsInitialized(PyInterpreterState *); + +// Export for 'pyexpat' shared extension +PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); + +extern int _PyImport_SetModuleString(const char *name, PyObject* module); + +extern void _PyImport_AcquireLock(PyInterpreterState *interp); +extern int _PyImport_ReleaseLock(PyInterpreterState *interp); + +extern int _PyImport_FixupBuiltin( + PyObject *mod, + const char *name, /* UTF-8 encoded string */ + PyObject *modules + ); +extern int _PyImport_FixupExtensionObject(PyObject*, PyObject *, + PyObject *, PyObject *); + +// Export for many shared extensions, like '_json' +PyAPI_FUNC(PyObject*) _PyImport_GetModuleAttr(PyObject *, PyObject *); + +// Export for many shared extensions, like '_datetime' +PyAPI_FUNC(PyObject*) _PyImport_GetModuleAttrString(const char *, const char *); + struct _import_runtime_state { /* The builtin modules (defined in config.c). */ @@ -15,19 +47,15 @@ struct _import_runtime_state { See PyInterpreterState.modules_by_index for more info. */ Py_ssize_t last_module_index; struct { - /* A thread state tied to the main interpreter, - used exclusively for when the extensions dict is access/modified - from an arbitrary thread. */ - PyThreadState main_tstate; - /* A lock to guard the dict. */ - PyThread_type_lock mutex; - /* A dict mapping (filename, name) to PyModuleDef for modules. + /* A lock to guard the cache. */ + PyMutex mutex; + /* The actual cache of (filename, name, PyModuleDef) for modules. Only legacy (single-phase init) extension modules are added and only if they support multiple initialization (m_size >- 0) or are imported in the main interpreter. This is initialized lazily in _PyImport_FixupExtensionObject(). Modules are added there and looked up in _imp.find_extension(). */ - PyObject *dict; + _Py_hashtable_t *hashtable; } extensions; /* Package context -- the full module name for package imports */ const char * pkgcontext; @@ -81,7 +109,7 @@ struct _import_state { }; #ifdef HAVE_DLOPEN -# include +# include // RTLD_NOW, RTLD_LAZY # if HAVE_DECL_RTLD_NOW # define _Py_DLOPEN_FLAGS RTLD_NOW # else @@ -165,16 +193,18 @@ struct _module_alias { const char *orig; /* ASCII encoded string */ }; -PyAPI_DATA(const struct _frozen *) _PyImport_FrozenBootstrap; -PyAPI_DATA(const struct _frozen *) _PyImport_FrozenStdlib; -PyAPI_DATA(const struct _frozen *) _PyImport_FrozenTest; +// Export these 3 symbols for test_ctypes +PyAPI_DATA(const struct _frozen*) _PyImport_FrozenBootstrap; +PyAPI_DATA(const struct _frozen*) _PyImport_FrozenStdlib; +PyAPI_DATA(const struct _frozen*) _PyImport_FrozenTest; + extern const struct _module_alias * _PyImport_FrozenAliases; -PyAPI_FUNC(int) _PyImport_CheckSubinterpIncompatibleExtensionAllowed( +extern int _PyImport_CheckSubinterpIncompatibleExtensionAllowed( const char *name); -// for testing +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); #ifdef __cplusplus diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h new file mode 100644 index 00000000000000..c8583582b358ac --- /dev/null +++ b/Include/internal/pycore_importdl.h @@ -0,0 +1,57 @@ +#ifndef Py_INTERNAL_IMPORTDL_H +#define Py_INTERNAL_IMPORTDL_H + +#include "patchlevel.h" // PY_MAJOR_VERSION + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +extern const char *_PyImport_DynLoadFiletab[]; + +extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); + +typedef PyObject *(*PyModInitFunction)(void); + +/* Max length of module suffix searched for -- accommodates "module.slb" */ +#define MAXSUFFIXSIZE 12 + +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; + +#ifdef _DEBUG +# define PYD_DEBUG_SUFFIX "_d" +#else +# define PYD_DEBUG_SUFFIX "" +#endif + +#ifdef Py_GIL_DISABLED +# define PYD_THREADING_TAG "t" +#else +# define PYD_THREADING_TAG "" +#endif + +#ifdef PYD_PLATFORM_TAG +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG +#else +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG +#endif + +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" +#define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" + +#else +typedef void (*dl_funcptr)(void); +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_IMPORTDL_H */ diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 4cbd14a61d4545..c86988234f6a05 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -22,7 +22,7 @@ struct pyruntimestate; #endif #define _PyStatus_OK() \ - (PyStatus){._type = _PyStatus_TYPE_OK,} + (PyStatus){._type = _PyStatus_TYPE_OK} /* other fields are set to 0 */ #define _PyStatus_ERR(ERR_MSG) \ (PyStatus){ \ @@ -30,7 +30,8 @@ struct pyruntimestate; .func = _PyStatus_GET_FUNC(), \ .err_msg = (ERR_MSG)} /* other fields are set to 0 */ -#define _PyStatus_NO_MEMORY() _PyStatus_ERR("memory allocation failed") +#define _PyStatus_NO_MEMORY_ERRMSG "memory allocation failed" +#define _PyStatus_NO_MEMORY() _PyStatus_ERR(_PyStatus_NO_MEMORY_ERRMSG) #define _PyStatus_EXIT(EXITCODE) \ (PyStatus){ \ ._type = _PyStatus_TYPE_EXIT, \ @@ -44,19 +45,23 @@ struct pyruntimestate; #define _PyStatus_UPDATE_FUNC(err) \ do { (err).func = _PyStatus_GET_FUNC(); } while (0) +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(void) _PyErr_SetFromPyStatus(PyStatus status); + + /* --- PyWideStringList ------------------------------------------------ */ #define _PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL} #ifndef NDEBUG -PyAPI_FUNC(int) _PyWideStringList_CheckConsistency(const PyWideStringList *list); +extern int _PyWideStringList_CheckConsistency(const PyWideStringList *list); #endif -PyAPI_FUNC(void) _PyWideStringList_Clear(PyWideStringList *list); -PyAPI_FUNC(int) _PyWideStringList_Copy(PyWideStringList *list, +extern void _PyWideStringList_Clear(PyWideStringList *list); +extern int _PyWideStringList_Copy(PyWideStringList *list, const PyWideStringList *list2); -PyAPI_FUNC(PyStatus) _PyWideStringList_Extend(PyWideStringList *list, +extern PyStatus _PyWideStringList_Extend(PyWideStringList *list, const PyWideStringList *list2); -PyAPI_FUNC(PyObject*) _PyWideStringList_AsList(const PyWideStringList *list); +extern PyObject* _PyWideStringList_AsList(const PyWideStringList *list); /* --- _PyArgv ---------------------------------------------------- */ @@ -68,28 +73,28 @@ typedef struct _PyArgv { wchar_t * const *wchar_argv; } _PyArgv; -PyAPI_FUNC(PyStatus) _PyArgv_AsWstrList(const _PyArgv *args, +extern PyStatus _PyArgv_AsWstrList(const _PyArgv *args, PyWideStringList *list); /* --- Helper functions ------------------------------------------- */ -PyAPI_FUNC(int) _Py_str_to_int( +extern int _Py_str_to_int( const char *str, int *result); -PyAPI_FUNC(const wchar_t*) _Py_get_xoption( +extern const wchar_t* _Py_get_xoption( const PyWideStringList *xoptions, const wchar_t *name); -PyAPI_FUNC(const char*) _Py_GetEnv( +extern const char* _Py_GetEnv( int use_environment, const char *name); -PyAPI_FUNC(void) _Py_get_env_flag( +extern void _Py_get_env_flag( int use_environment, int *flag, const char *name); /* Py_GetArgcArgv() helper */ -PyAPI_FUNC(void) _Py_ClearArgcArgv(void); +extern void _Py_ClearArgcArgv(void); /* --- _PyPreCmdline ------------------------------------------------- */ @@ -122,7 +127,9 @@ extern PyStatus _PyPreCmdline_Read(_PyPreCmdline *cmdline, /* --- PyPreConfig ----------------------------------------------- */ +// Export for '_testembed' program PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(PyPreConfig *preconfig); + extern void _PyPreConfig_InitFromConfig( PyPreConfig *preconfig, const PyConfig *config); @@ -146,7 +153,9 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; +// Export for '_testembed' program PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); + extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); @@ -161,16 +170,16 @@ extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, const _PyArgv *args); -PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); -PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); extern void _Py_DumpPathConfig(PyThreadState *tstate); -PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject(void); - /* --- Function used for testing ---------------------------------- */ +// Export these functions for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); +PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); +PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject(void); PyAPI_FUNC(PyObject*) _Py_GetConfigsAsDict(void); #ifdef __cplusplus diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 9fb3952227af18..eae8371ef7f9b8 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -1,12 +1,11 @@ - #ifndef Py_INTERNAL_INSTRUMENT_H #define Py_INTERNAL_INSTRUMENT_H +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif -#include "pycore_bitutils.h" // _Py_popcount32 -#include "pycore_frame.h" - -#include "cpython/code.h" +#include "pycore_frame.h" // _PyInterpreterFrame #ifdef __cplusplus extern "C" { @@ -28,7 +27,8 @@ extern "C" { #define PY_MONITORING_EVENT_BRANCH 8 #define PY_MONITORING_EVENT_STOP_ITERATION 9 -#define PY_MONITORING_INSTRUMENTED_EVENTS 10 +#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ + ((ev) < _PY_MONITORING_LOCAL_EVENTS) /* Other events, mainly exceptions */ @@ -36,12 +36,13 @@ extern "C" { #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 #define PY_MONITORING_EVENT_PY_UNWIND 12 #define PY_MONITORING_EVENT_PY_THROW 13 +#define PY_MONITORING_EVENT_RERAISE 14 /* Ancilliary events */ -#define PY_MONITORING_EVENT_C_RETURN 14 -#define PY_MONITORING_EVENT_C_RAISE 15 +#define PY_MONITORING_EVENT_C_RETURN 15 +#define PY_MONITORING_EVENT_C_RAISE 16 typedef uint32_t _PyMonitoringEventSet; @@ -62,6 +63,8 @@ typedef uint32_t _PyMonitoringEventSet; PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); +int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events); +int _PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events); extern int _Py_call_instrumentation(PyThreadState *tstate, int event, @@ -88,10 +91,6 @@ extern int _Py_call_instrumentation_2args(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern void -_Py_call_instrumentation_exc0(PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); - extern void _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 527b2121148f4c..2a683196eeced3 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -8,29 +8,30 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include +#include // bool #include "pycore_ast_state.h" // struct ast_state #include "pycore_atexit.h" // struct atexit_state -#include "pycore_atomic.h" // _Py_atomic_address #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache #include "pycore_context.h" // struct _Py_context_state +#include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_dict_state.h" // struct _Py_dict_state #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_function.h" // FUNC_MAX_WATCHERS -#include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state -#include "pycore_global_objects.h" // struct _Py_interp_static_objects +#include "pycore_genobject.h" // struct _Py_async_gen_state +#include "pycore_global_objects.h"// struct _Py_interp_cached_objects #include "pycore_import.h" // struct _import_state -#include "pycore_instruments.h" // PY_MONITORING_EVENTS +#include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state -#include "pycore_object_state.h" // struct _py_object_state -#include "pycore_obmalloc.h" // struct obmalloc_state +#include "pycore_object_state.h" // struct _py_object_state +#include "pycore_obmalloc.h" // struct _obmalloc_state +#include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state -#include "pycore_typeobject.h" // struct type_cache +#include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state @@ -39,6 +40,10 @@ struct _Py_long_state { int max_str_digits; }; + +/* cross-interpreter data registry */ + + /* interpreter state */ /* PyInterpreterState holds the global state for one of the runtime's @@ -48,15 +53,32 @@ struct _Py_long_state { */ struct _is { + /* This struct countains the eval_breaker, + * which is by far the hottest field in this struct + * and should be placed at the beginning. */ + struct _ceval_state ceval; + PyInterpreterState *next; - uint64_t monitoring_version; - uint64_t last_restart_version; + int64_t id; + int64_t id_refcount; + int requires_idref; + PyThread_type_lock id_mutex; + + /* Has been initialized to a safe state. + + In order to be effective, this must be set to 0 during or right + after allocation. */ + int _initialized; + int finalizing; + uintptr_t last_restart_version; struct pythreads { uint64_t next_unique_id; /* The linked list of threads, newest first. */ PyThreadState *head; + /* The thread currently executing in the __main__ module, if any. */ + PyThreadState *main; /* Used in Modules/_threadmodule.c. */ long count; /* Support for runtime thread stack size tuning. @@ -71,30 +93,45 @@ struct _is { Get runtime from tstate: tstate->interp->runtime. */ struct pyruntimestate *runtime; - int64_t id; - int64_t id_refcount; - int requires_idref; - PyThread_type_lock id_mutex; + /* Set by Py_EndInterpreter(). - /* Has been initialized to a safe state. + Use _PyInterpreterState_GetFinalizing() + and _PyInterpreterState_SetFinalizing() + to access it, don't access it directly. */ + PyThreadState* _finalizing; + /* The ID of the OS thread in which we are finalizing. */ + unsigned long _finalizing_id; - In order to be effective, this must be set to 0 during or right - after allocation. */ - int _initialized; - int finalizing; + struct _gc_runtime_state gc; - struct _obmalloc_state obmalloc; + /* The following fields are here to avoid allocation during init. + The data is exposed through PyInterpreterState pointer fields. + These fields should not be accessed directly outside of init. - struct _ceval_state ceval; - struct _gc_runtime_state gc; + All other PyInterpreterState pointer fields are populated when + needed and default to NULL. - struct _import_state imports; + For now there are some exceptions to that rule, which require + allocation during init. These will be addressed on a case-by-case + basis. Also see _PyRuntimeState regarding the various mutex fields. + */ // Dictionary of the sys module PyObject *sysdict; + // Dictionary of the builtins module PyObject *builtins; + struct _import_state imports; + + /* The per-interpreter GIL, which might not be used. */ + struct _gil_runtime_state _gil; + + /* ---------- IMPORTANT --------------------------- + The fields above this line are declared as early as + possible to facilitate out-of-process observability + tools. */ + PyObject *codec_search_path; PyObject *codec_search_cache; PyObject *codec_error_registry; @@ -117,6 +154,9 @@ struct _is { Py_ssize_t co_extra_user_count; freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; + /* cross-interpreter data and utils */ + struct _xi_state xi; + #ifdef HAVE_FORK PyObject *before_forkers; PyObject *after_forkers_parent; @@ -126,6 +166,8 @@ struct _is { struct _warnings_runtime_state warnings; struct atexit_state atexit; + struct _obmalloc_state obmalloc; + PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; PyCode_WatchCallback code_watchers[CODE_MAX_WATCHERS]; @@ -152,37 +194,26 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; - PyCodeObject *interpreter_trampoline; + _PyOptimizerObject *optimizer; + _PyExecutorObject *executor_list_head; + uint16_t optimizer_resume_threshold; + uint16_t optimizer_backedge_threshold; + uint32_t next_func_version; - _Py_Monitors monitors; - bool f_opcode_trace_set; + _Py_GlobalMonitors monitors; bool sys_profile_initialized; bool sys_trace_initialized; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ - PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][PY_MONITORING_EVENTS]; + PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; - /* The following fields are here to avoid allocation during init. - The data is exposed through PyInterpreterState pointer fields. - These fields should not be accessed directly outside of init. - - All other PyInterpreterState pointer fields are populated when - needed and default to NULL. - - For now there are some exceptions to that rule, which require - allocation during init. These will be addressed on a case-by-case - basis. Also see _PyRuntimeState regarding the various mutex fields. - */ - - /* The per-interpreter GIL, which might not be used. */ - struct _gil_runtime_state _gil; - /* the initial PyInterpreterState.threads.head */ - PyThreadState _initial_thread; + _PyThreadStateImpl _initial_thread; + Py_ssize_t _interactive_src_count; }; @@ -191,26 +222,112 @@ struct _is { extern void _PyInterpreterState_Clear(PyThreadState *tstate); -/* cross-interpreter data registry */ +static inline PyThreadState* +_PyInterpreterState_GetFinalizing(PyInterpreterState *interp) { + return (PyThreadState*)_Py_atomic_load_ptr_relaxed(&interp->_finalizing); +} -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ +static inline unsigned long +_PyInterpreterState_GetFinalizingID(PyInterpreterState *interp) { + return _Py_atomic_load_ulong_relaxed(&interp->_finalizing_id); +} -struct _xidregitem; +static inline void +_PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tstate) { + _Py_atomic_store_ptr_relaxed(&interp->_finalizing, tstate); + if (tstate == NULL) { + _Py_atomic_store_ulong_relaxed(&interp->_finalizing_id, 0); + } + else { + // XXX Re-enable this assert once gh-109860 is fixed. + //assert(tstate->thread_id == PyThread_get_thread_ident()); + _Py_atomic_store_ulong_relaxed(&interp->_finalizing_id, + tstate->thread_id); + } +} -struct _xidregitem { - struct _xidregitem *prev; - struct _xidregitem *next; - PyObject *cls; // weakref to a PyTypeObject - crossinterpdatafunc getdata; -}; -PyAPI_FUNC(PyInterpreterState*) _PyInterpreterState_LookUpID(int64_t); +// Export for the _xxinterpchannels module. +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); + +extern int _PyInterpreterState_IDInitref(PyInterpreterState *); +extern int _PyInterpreterState_IDIncref(PyInterpreterState *); +extern void _PyInterpreterState_IDDecref(PyInterpreterState *); + +extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); + +// Get a copy of the current interpreter configuration. +// +// Return 0 on success. Raise an exception and return -1 on error. +// +// The caller must initialize 'config', using PyConfig_InitPythonConfig() +// for example. +// +// Python must be preinitialized to call this method. +// The caller must hold the GIL. +// +// Once done with the configuration, PyConfig_Clear() must be called to clear +// it. +// +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( + struct PyConfig *config); + +// Set the configuration of the current interpreter. +// +// This function should be called during or just after the Python +// initialization. +// +// Update the sys module with the new configuration. If the sys module was +// modified directly after the Python initialization, these changes are lost. +// +// Some configuration like faulthandler or warnoptions can be updated in the +// configuration, but don't reconfigure Python (don't enable/disable +// faulthandler and don't reconfigure warnings filters). +// +// Return 0 on success. Raise an exception and return -1 on error. +// +// The configuration should come from _PyInterpreterState_GetConfigCopy(). +// +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(int) _PyInterpreterState_SetConfig( + const struct PyConfig *config); + + +/* +Runtime Feature Flags + +Each flag indicate whether or not a specific runtime feature +is available in a given context. For example, forking the process +might not be allowed in the current interpreter (i.e. os.fork() would fail). +*/ + +/* Set if the interpreter share obmalloc runtime state + with the main interpreter. */ +#define Py_RTFLAGS_USE_MAIN_OBMALLOC (1UL << 5) + +/* Set if import should check a module for subinterpreter support. */ +#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8) + +/* Set if threads are allowed. */ +#define Py_RTFLAGS_THREADS (1UL << 10) + +/* Set if daemon threads are allowed. */ +#define Py_RTFLAGS_DAEMON_THREADS (1UL << 11) + +/* Set if os.fork() is allowed. */ +#define Py_RTFLAGS_FORK (1UL << 15) + +/* Set if os.exec*() is allowed. */ +#define Py_RTFLAGS_EXEC (1UL << 16) + +extern int _PyInterpreterState_HasFeature(PyInterpreterState *interp, + unsigned long feature); + +PyAPI_FUNC(PyStatus) _PyInterpreterState_New( + PyThreadState *tstate, + PyInterpreterState **pinterp); -PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); -PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); -PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interpreteridobject.h b/Include/internal/pycore_interpreteridobject.h deleted file mode 100644 index 804831e76deaea..00000000000000 --- a/Include/internal/pycore_interpreteridobject.h +++ /dev/null @@ -1,22 +0,0 @@ -/* Interpreter ID Object */ - -#ifndef Py_INTERNAL_INTERPRETERIDOBJECT_H -#define Py_INTERNAL_INTERPRETERIDOBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type; - -PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t); -PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); -PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *); - -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_INTERPRETERIDOBJECT_H diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 3902059a04b9da..3a8dd95cff8e5d 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -1,4 +1,9 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py +#ifndef Py_INTERNAL_INTRINSIC_H +#define Py_INTERNAL_INTRINSIC_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif /* Unary Functions: */ #define INTRINSIC_1_INVALID 0 @@ -8,17 +13,38 @@ #define INTRINSIC_ASYNC_GEN_WRAP 4 #define INTRINSIC_UNARY_POSITIVE 5 #define INTRINSIC_LIST_TO_TUPLE 6 +#define INTRINSIC_TYPEVAR 7 +#define INTRINSIC_PARAMSPEC 8 +#define INTRINSIC_TYPEVARTUPLE 9 +#define INTRINSIC_SUBSCRIPT_GENERIC 10 +#define INTRINSIC_TYPEALIAS 11 -#define MAX_INTRINSIC_1 6 +#define MAX_INTRINSIC_1 11 /* Binary Functions: */ #define INTRINSIC_2_INVALID 0 #define INTRINSIC_PREP_RERAISE_STAR 1 +#define INTRINSIC_TYPEVAR_WITH_BOUND 2 +#define INTRINSIC_TYPEVAR_WITH_CONSTRAINTS 3 +#define INTRINSIC_SET_FUNCTION_TYPE_PARAMS 4 + +#define MAX_INTRINSIC_2 4 + +typedef PyObject *(*intrinsic_func1)(PyThreadState* tstate, PyObject *value); +typedef PyObject *(*intrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); + +typedef struct { + intrinsic_func1 func; + const char *name; +} intrinsic_func1_info; + +typedef struct { + intrinsic_func2 func; + const char *name; +} intrinsic_func2_info; -#define MAX_INTRINSIC_2 1 +extern const intrinsic_func1_info _PyIntrinsics_UnaryFunctions[]; +extern const intrinsic_func2_info _PyIntrinsics_BinaryFunctions[]; -typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); -typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); -extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; -extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; +#endif // !Py_INTERNAL_INTRINSIC_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 2fcbe12cd6559e..55d67b32bc8a63 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,7 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "listobject.h" // _PyList_CAST() + +extern PyObject* _PyList_Extend(PyListObject *, PyObject *); +extern void _PyList_DebugMallocStats(FILE *out); /* runtime lifecycle */ diff --git a/Include/internal/pycore_llist.h b/Include/internal/pycore_llist.h new file mode 100644 index 00000000000000..5fd261da05fa5d --- /dev/null +++ b/Include/internal/pycore_llist.h @@ -0,0 +1,107 @@ +// A doubly-linked list that can be embedded in a struct. +// +// Usage: +// struct llist_node head = LLIST_INIT(head); +// typedef struct { +// ... +// struct llist_node node; +// ... +// } MyObj; +// +// llist_insert_tail(&head, &obj->node); +// llist_remove(&obj->node); +// +// struct llist_node *node; +// llist_for_each(node, &head) { +// MyObj *obj = llist_data(node, MyObj, node); +// ... +// } +// + +#ifndef Py_INTERNAL_LLIST_H +#define Py_INTERNAL_LLIST_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +struct llist_node { + struct llist_node *next; + struct llist_node *prev; +}; + +// Get the struct containing a node. +#define llist_data(node, type, member) \ + (type*)((char*)node - offsetof(type, member)) + +// Iterate over a list. +#define llist_for_each(node, head) \ + for (node = (head)->next; node != (head); node = node->next) + +// Iterate over a list, but allow removal of the current node. +#define llist_for_each_safe(node, head) \ + for (struct llist_node *_next = (node = (head)->next, node->next); \ + node != (head); node = _next, _next = node->next) + +#define LLIST_INIT(head) { &head, &head } + +static inline void +llist_init(struct llist_node *head) +{ + head->next = head; + head->prev = head; +} + +// Returns 1 if the list is empty, 0 otherwise. +static inline int +llist_empty(struct llist_node *head) +{ + return head->next == head; +} + +// Appends to the tail of the list. +static inline void +llist_insert_tail(struct llist_node *head, struct llist_node *node) +{ + node->prev = head->prev; + node->next = head; + head->prev->next = node; + head->prev = node; +} + +// Remove a node from the list. +static inline void +llist_remove(struct llist_node *node) +{ + struct llist_node *prev = node->prev; + struct llist_node *next = node->next; + prev->next = next; + next->prev = prev; + node->prev = NULL; + node->next = NULL; +} + +// Append all nodes from head2 onto head1. head2 is left empty. +static inline void +llist_concat(struct llist_node *head1, struct llist_node *head2) +{ + if (!llist_empty(head2)) { + head1->prev->next = head2->next; + head2->next->prev = head1->prev; + + head1->prev = head2->prev; + head2->prev->next = head1; + llist_init(head2); + } +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_LLIST_H */ diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h new file mode 100644 index 00000000000000..03ad1c9fd584b5 --- /dev/null +++ b/Include/internal/pycore_lock.h @@ -0,0 +1,219 @@ +// Lightweight locks and other synchronization mechanisms. +// +// These implementations are based on WebKit's WTF::Lock. See +// https://webkit.org/blog/6161/locking-in-webkit/ for a description of the +// design. +#ifndef Py_INTERNAL_LOCK_H +#define Py_INTERNAL_LOCK_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_time.h" // _PyTime_t + + +// A mutex that occupies one byte. The lock can be zero initialized. +// +// Only the two least significant bits are used. The remaining bits should be +// zero: +// 0b00: unlocked +// 0b01: locked +// 0b10: unlocked and has parked threads +// 0b11: locked and has parked threads +// +// Typical initialization: +// PyMutex m = (PyMutex){0}; +// +// Typical usage: +// PyMutex_Lock(&m); +// ... +// PyMutex_Unlock(&m); + +// NOTE: In Py_GIL_DISABLED builds, `struct _PyMutex` is defined in Include/object.h. +// The Py_GIL_DISABLED builds need the definition in Include/object.h for the +// `ob_mutex` field in PyObject. For the default (non-free-threaded) build, +// we define the struct here to avoid exposing it in the public API. +#ifndef Py_GIL_DISABLED +struct _PyMutex { uint8_t v; }; +#endif + +typedef struct _PyMutex PyMutex; + +#define _Py_UNLOCKED 0 +#define _Py_LOCKED 1 +#define _Py_HAS_PARKED 2 +#define _Py_ONCE_INITIALIZED 4 + +// (private) slow path for locking the mutex +PyAPI_FUNC(void) _PyMutex_LockSlow(PyMutex *m); + +// (private) slow path for unlocking the mutex +PyAPI_FUNC(void) _PyMutex_UnlockSlow(PyMutex *m); + +static inline int +PyMutex_LockFast(uint8_t *lock_bits) +{ + uint8_t expected = _Py_UNLOCKED; + return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); +} + +// Locks the mutex. +// +// If the mutex is currently locked, the calling thread will be parked until +// the mutex is unlocked. If the current thread holds the GIL, then the GIL +// will be released while the thread is parked. +static inline void +PyMutex_Lock(PyMutex *m) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { + _PyMutex_LockSlow(m); + } +} + +// Unlocks the mutex. +static inline void +PyMutex_Unlock(PyMutex *m) +{ + uint8_t expected = _Py_LOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_UNLOCKED)) { + _PyMutex_UnlockSlow(m); + } +} + +// Checks if the mutex is currently locked. +static inline int +PyMutex_IsLocked(PyMutex *m) +{ + return (_Py_atomic_load_uint8(&m->v) & _Py_LOCKED) != 0; +} + +// Re-initializes the mutex after a fork to the unlocked state. +static inline void +_PyMutex_at_fork_reinit(PyMutex *m) +{ + memset(m, 0, sizeof(*m)); +} + +typedef enum _PyLockFlags { + // Do not detach/release the GIL when waiting on the lock. + _Py_LOCK_DONT_DETACH = 0, + + // Detach/release the GIL while waiting on the lock. + _PY_LOCK_DETACH = 1, + + // Handle signals if interrupted while waiting on the lock. + _PY_LOCK_HANDLE_SIGNALS = 2, +} _PyLockFlags; + +// Lock a mutex with an optional timeout and additional options. See +// _PyLockFlags for details. +extern PyLockStatus +_PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout_ns, _PyLockFlags flags); + +// Lock a mutex with aditional options. See _PyLockFlags for details. +static inline void +PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { + _PyMutex_LockTimed(m, -1, flags); + } +} + +// Unlock a mutex, returns 0 if the mutex is not locked (used for improved +// error messages). +extern int _PyMutex_TryUnlock(PyMutex *m); + + +// PyEvent is a one-time event notification +typedef struct { + uint8_t v; +} PyEvent; + +// Set the event and notify any waiting threads. +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(void) _PyEvent_Notify(PyEvent *evt); + +// Wait for the event to be set. If the event is already set, then this returns +// immediately. +PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); + +// Wait for the event to be set, or until the timeout expires. If the event is +// already set, then this returns immediately. Returns 1 if the event was set, +// and 0 if the timeout expired or thread was interrupted. +PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns); + + +// _PyRawMutex implements a word-sized mutex that that does not depend on the +// parking lot API, and therefore can be used in the parking lot +// implementation. +// +// The mutex uses a packed representation: the least significant bit is used to +// indicate whether the mutex is locked or not. The remaining bits are either +// zero or a pointer to a `struct raw_mutex_entry` (see lock.c). +typedef struct { + uintptr_t v; +} _PyRawMutex; + +// Slow paths for lock/unlock +extern void _PyRawMutex_LockSlow(_PyRawMutex *m); +extern void _PyRawMutex_UnlockSlow(_PyRawMutex *m); + +static inline void +_PyRawMutex_Lock(_PyRawMutex *m) +{ + uintptr_t unlocked = _Py_UNLOCKED; + if (_Py_atomic_compare_exchange_uintptr(&m->v, &unlocked, _Py_LOCKED)) { + return; + } + _PyRawMutex_LockSlow(m); +} + +static inline void +_PyRawMutex_Unlock(_PyRawMutex *m) +{ + uintptr_t locked = _Py_LOCKED; + if (_Py_atomic_compare_exchange_uintptr(&m->v, &locked, _Py_UNLOCKED)) { + return; + } + _PyRawMutex_UnlockSlow(m); +} + +// A data structure that can be used to run initialization code once in a +// thread-safe manner. The C++11 equivalent is std::call_once. +typedef struct { + uint8_t v; +} _PyOnceFlag; + +// Type signature for one-time initialization functions. The function should +// return 0 on success and -1 on failure. +typedef int _Py_once_fn_t(void *arg); + +// (private) slow path for one time initialization +PyAPI_FUNC(int) +_PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg); + +// Calls `fn` once using `flag`. The `arg` is passed to the call to `fn`. +// +// Returns 0 on success and -1 on failure. +// +// If `fn` returns 0 (success), then subsequent calls immediately return 0. +// If `fn` returns -1 (failure), then subsequent calls will retry the call. +static inline int +_PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) +{ + if (_Py_atomic_load_uint8(&flag->v) == _Py_ONCE_INITIALIZED) { + return 0; + } + return _PyOnceFlag_CallOnceSlow(flag, fn, arg); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_LOCK_H */ diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index fe86581e81f6b5..ec27df9e416c58 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -8,7 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_global_objects.h" // _PY_NSMALLNEGINTS +#include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_global_objects.h"// _PY_NSMALLNEGINTS #include "pycore_runtime.h" // _PyRuntime /* @@ -46,6 +47,16 @@ extern "C" { # error "_PY_LONG_DEFAULT_MAX_STR_DIGITS smaller than threshold." #endif +// _PyLong_NumBits. Return the number of bits needed to represent the +// absolute value of a long. For example, this returns 1 for 1 and -1, 2 +// for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. +// v must not be NULL, and must be a normalized long. +// (size_t)-1 is returned and OverflowError set if the true result doesn't +// fit in a size_t. +// +// Export for 'math' shared extension. +PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); + /* runtime lifecycle */ @@ -63,51 +74,99 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -// Return a borrowed reference to the zero singleton. +// Return a reference to the immortal zero singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetZero(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS]; } -// Return a borrowed reference to the one singleton. +// Return a reference to the immortal one singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetOne(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+1]; } static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) { - return Py_NewRef((PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]); + return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]; } -PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); -PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right); -PyObject *_PyLong_Subtract(PyLongObject *left, PyLongObject *right); - -/* Used by Python/mystrtoul.c, _PyBytes_FromHex(), - _PyBytes_DecodeEscape(), etc. */ +// _PyLong_Frexp returns a double x and an exponent e such that the +// true value is approximately equal to x * 2**e. e is >= 0. x is +// 0.0 if and only if the input is 0 (in which case, e and x are both +// zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is +// possible if the number of bits doesn't fit into a Py_ssize_t, sets +// OverflowError and returns -1.0 for x, 0 for e. +// +// Export for 'math' shared extension +PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); + +extern PyObject* _PyLong_FromBytes(const char *, Py_ssize_t, int); + +// _PyLong_DivmodNear. Given integers a and b, compute the nearest +// integer q to the exact quotient a / b, rounding to the nearest even integer +// in the case of a tie. Return (q, r), where r = a - q*b. The remainder r +// will satisfy abs(r) <= abs(b)/2, with equality possible only if q is +// even. +// +// Export for '_datetime' shared extension. +PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); + +// _PyLong_Format: Convert the long to a string object with given base, +// appending a base prefix of 0[box] if base is 2, 8 or 16. +// Export for '_tkinter' shared extension. +PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); + +// Export for 'math' shared extension +PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t); + +// Export for 'math' shared extension +PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t); + +extern PyObject* _PyLong_Add(PyLongObject *left, PyLongObject *right); +extern PyObject* _PyLong_Multiply(PyLongObject *left, PyLongObject *right); +extern PyObject* _PyLong_Subtract(PyLongObject *left, PyLongObject *right); + +// Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( +extern int _PyLong_FormatAdvancedWriter( _PyUnicodeWriter *writer, PyObject *obj, PyObject *format_spec, Py_ssize_t start, Py_ssize_t end); -PyAPI_FUNC(int) _PyLong_FormatWriter( +extern int _PyLong_FormatWriter( _PyUnicodeWriter *writer, PyObject *obj, int base, int alternate); -PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( +extern char* _PyLong_FormatBytesWriter( _PyBytesWriter *writer, char *str, PyObject *obj, int base, int alternate); +// Argument converters used by Argument Clinic + +// Export for 'select' shared extension (Argument Clinic code) +PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); + +// Export for '_testclinic' shared extension (Argument Clinic code) +PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); + +// Export for '_blake2' shared extension (Argument Clinic code) +PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); + +// Export for '_blake2' shared extension (Argument Clinic code) +PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); + +// Export for '_testclinic' shared extension (Argument Clinic code) +PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); + /* Long value tag bits: * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1. * 2: Reserved for immortality bit @@ -118,6 +177,21 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( #define SIGN_NEGATIVE 2 #define NON_SIZE_BITS 3 +/* The functions _PyLong_IsCompact and _PyLong_CompactValue are defined + * in Include/cpython/longobject.h, since they need to be inline. + * + * "Compact" values have at least one bit to spare, + * so that addition and subtraction can be performed on the values + * without risk of overflow. + * + * The inline functions need tag bits. + * For readability, rather than do `#define SIGN_MASK _PyLong_SIGN_MASK` + * we define them to the numbers in both places and then assert that + * they're the same. + */ +static_assert(SIGN_MASK == _PyLong_SIGN_MASK, "SIGN_MASK does not match _PyLong_SIGN_MASK"); +static_assert(NON_SIZE_BITS == _PyLong_NON_SIZE_BITS, "NON_SIZE_BITS does not match _PyLong_NON_SIZE_BITS"); + /* All *compact" values are guaranteed to fit into * a Py_ssize_t with at least one bit to spare. * In other words, for 64 bit machines, compact @@ -131,11 +205,6 @@ _PyLong_IsNonNegativeCompact(const PyLongObject* op) { return op->long_value.lv_tag <= (1 << NON_SIZE_BITS); } -static inline int -_PyLong_IsCompact(const PyLongObject* op) { - assert(PyLong_Check(op)); - return op->long_value.lv_tag < (2 << NON_SIZE_BITS); -} static inline int _PyLong_BothAreCompact(const PyLongObject* a, const PyLongObject* b) { @@ -144,21 +213,6 @@ _PyLong_BothAreCompact(const PyLongObject* a, const PyLongObject* b) { return (a->long_value.lv_tag | b->long_value.lv_tag) < (2 << NON_SIZE_BITS); } -/* Returns a *compact* value, iff `_PyLong_IsCompact` is true for `op`. - * - * "Compact" values have at least one bit to spare, - * so that addition and subtraction can be performed on the values - * without risk of overflow. - */ -static inline Py_ssize_t -_PyLong_CompactValue(const PyLongObject *op) -{ - assert(PyLong_Check(op)); - assert(_PyLong_IsCompact(op)); - Py_ssize_t sign = 1 - (op->long_value.lv_tag & SIGN_MASK); - return sign * (Py_ssize_t)op->long_value.ob_digit[0]; -} - static inline bool _PyLong_IsZero(const PyLongObject *op) { @@ -245,7 +299,7 @@ _PyLong_FlipSign(PyLongObject *op) { #define _PyLong_DIGIT_INIT(val) \ { \ - .ob_base = _PyObject_HEAD_INIT(&PyLong_Type) \ + .ob_base = _PyObject_HEAD_INIT(&PyLong_Type), \ .long_value = { \ .lv_tag = TAG_FROM_SIGN_AND_SIZE( \ (val) == 0 ? 0 : ((val) < 0 ? -1 : 1), \ diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h index fe19e3f9611a16..62e204fcbf6533 100644 --- a/Include/internal/pycore_memoryobject.h +++ b/Include/internal/pycore_memoryobject.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +extern PyTypeObject _PyManagedBuffer_Type; + PyObject * _PyMemoryView_FromBufferProc(PyObject *v, int flags, getbufferproc bufferproc); diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h new file mode 100644 index 00000000000000..c29dc82a42762a --- /dev/null +++ b/Include/internal/pycore_mimalloc.h @@ -0,0 +1,19 @@ +#ifndef Py_INTERNAL_MIMALLOC_H +#define Py_INTERNAL_MIMALLOC_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#if defined(MIMALLOC_H) || defined(MIMALLOC_TYPES_H) +# error "pycore_mimalloc.h must be included before mimalloc.h" +#endif + +#include "pycore_pymem.h" +#define MI_DEBUG_UNINIT PYMEM_CLEANBYTE +#define MI_DEBUG_FREED PYMEM_DEADBYTE +#define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE + +#include "mimalloc.h" + +#endif // Py_INTERNAL_MIMALLOC_H diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h new file mode 100644 index 00000000000000..3d3cd6722528e9 --- /dev/null +++ b/Include/internal/pycore_modsupport.h @@ -0,0 +1,125 @@ +#ifndef Py_INTERNAL_MODSUPPORT_H +#define Py_INTERNAL_MODSUPPORT_H + +#include "pycore_lock.h" // _PyOnceFlag + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +extern int _PyArg_NoKwnames(const char *funcname, PyObject *kwnames); +#define _PyArg_NoKwnames(funcname, kwnames) \ + ((kwnames) == NULL || _PyArg_NoKwnames((funcname), (kwnames))) + +// Export for '_bz2' shared extension +PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args); +#define _PyArg_NoPositional(funcname, args) \ + ((args) == NULL || _PyArg_NoPositional((funcname), (args))) + +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kwargs); +#define _PyArg_NoKeywords(funcname, kwargs) \ + ((kwargs) == NULL || _PyArg_NoKeywords((funcname), (kwargs))) + +// Export for 'zlib' shared extension +PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, + Py_ssize_t, Py_ssize_t); +#define _Py_ANY_VARARGS(n) ((n) == PY_SSIZE_T_MAX) +#define _PyArg_CheckPositional(funcname, nargs, min, max) \ + ((!_Py_ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ + || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) + +extern PyObject ** _Py_VaBuildStack( + PyObject **small_stack, + Py_ssize_t small_stack_len, + const char *format, + va_list va, + Py_ssize_t *p_nargs); + +extern PyObject* _PyModule_CreateInitialized(PyModuleDef*, int apiver); + +// Export for '_curses' shared extension +PyAPI_FUNC(int) _PyArg_ParseStack( + PyObject *const *args, + Py_ssize_t nargs, + const char *format, + ...); + +extern int _PyArg_UnpackStack( + PyObject *const *args, + Py_ssize_t nargs, + const char *name, + Py_ssize_t min, + Py_ssize_t max, + ...); + +// Export for '_heapq' shared extension +PyAPI_FUNC(void) _PyArg_BadArgument( + const char *fname, + const char *displayname, + const char *expected, + PyObject *arg); + +// --- _PyArg_Parser API --------------------------------------------------- + +typedef struct _PyArg_Parser { + const char *format; + const char * const *keywords; + const char *fname; + const char *custom_msg; + _PyOnceFlag once; /* atomic one-time initialization flag */ + int is_kwtuple_owned; /* does this parser own the kwtuple object? */ + int pos; /* number of positional-only arguments */ + int min; /* minimal number of arguments */ + int max; /* maximal number of positional arguments */ + PyObject *kwtuple; /* tuple of keyword parameter names */ + struct _PyArg_Parser *next; +} _PyArg_Parser; + +// Export for '_testclinic' shared extension +PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, + struct _PyArg_Parser *, ...); + +// Export for '_dbm' shared extension +PyAPI_FUNC(int) _PyArg_ParseStackAndKeywords( + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames, + struct _PyArg_Parser *, + ...); + +// Export for 'math' shared extension +PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwargs, + PyObject *kwnames, + struct _PyArg_Parser *parser, + int minpos, + int maxpos, + int minkw, + PyObject **buf); +#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ + (((minkw) == 0 && (kwargs) == NULL && (kwnames) == NULL && \ + (minpos) <= (nargs) && (nargs) <= (maxpos) && (args) != NULL) ? (args) : \ + _PyArg_UnpackKeywords((args), (nargs), (kwargs), (kwnames), (parser), \ + (minpos), (maxpos), (minkw), (buf))) + +// Export for '_testclinic' shared extension +PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject *kwnames, + struct _PyArg_Parser *parser, + int minpos, int maxpos, int minkw, + int vararg, PyObject **buf); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MODSUPPORT_H + diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 15a1bcb6ae5163..5644bbe5e0552b 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -8,6 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +extern void _PyModule_Clear(PyObject *); +extern void _PyModule_ClearDict(PyObject *); +extern int _PyModuleSpec_IsInitializing(PyObject *); + +extern int _PyModule_IsExtension(PyObject *obj); + typedef struct { PyObject_HEAD PyObject *md_dict; @@ -33,7 +39,7 @@ static inline PyObject* _PyModule_GetDict(PyObject *mod) { PyObject *dict = ((PyModuleObject *)mod) -> md_dict; // _PyModule_GetDict(mod) must not be used after calling module_clear(mod) assert(dict != NULL); - return dict; + return dict; // borrowed reference } PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress); diff --git a/Include/internal/pycore_namespace.h b/Include/internal/pycore_namespace.h index cb76f040693d10..f165cf15319a59 100644 --- a/Include/internal/pycore_namespace.h +++ b/Include/internal/pycore_namespace.h @@ -10,9 +10,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -PyAPI_DATA(PyTypeObject) _PyNamespace_Type; +extern PyTypeObject _PyNamespace_Type; -PyAPI_FUNC(PyObject *) _PyNamespace_New(PyObject *kwds); +// Export for '_testmultiphase' shared extension +PyAPI_FUNC(PyObject*) _PyNamespace_New(PyObject *kwds); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 500b3eece68055..f413b8451e5ab4 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -10,9 +10,41 @@ extern "C" { #include #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall() #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_runtime.h" // _PyRuntime + +/* Check if an object is consistent. For example, ensure that the reference + counter is greater than or equal to 1, and ensure that ob_type is not NULL. + + Call _PyObject_AssertFailed() if the object is inconsistent. + + If check_content is zero, only check header fields: reduce the overhead. + + The function always return 1. The return value is just here to be able to + write: + + assert(_PyObject_CheckConsistency(obj, 1)); */ +extern int _PyObject_CheckConsistency(PyObject *op, int check_content); + +extern void _PyDebugAllocatorStats(FILE *out, const char *block_name, + int num_blocks, size_t sizeof_block); + +extern void _PyObject_DebugTypeStats(FILE *out); + +#ifdef Py_TRACE_REFS +// Forget a reference registered by _Py_NewReference(). Function called by +// _Py_Dealloc(). +// +// On a free list, the function can be used before modifying an object to +// remove the object from traced objects. Then _Py_NewReference() or +// _Py_NewReferenceNoTotal() should be called again on the object to trace +// it again. +extern void _Py_ForgetReference(PyObject *); +#endif + +// Export for shared _testinternalcapi extension +PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); /* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid designated initializer conflicts in C++20. If we use the deinition in @@ -22,19 +54,26 @@ extern "C" { Furthermore, we can't use designated initializers in Extensions since these are not supported pre-C++20. Thus, keeping an internal copy here is the most backwards compatible solution */ +#if defined(Py_GIL_DISABLED) +#define _PyObject_HEAD_INIT(type) \ + { \ + .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \ + .ob_type = (type) \ + } +#else #define _PyObject_HEAD_INIT(type) \ { \ - _PyObject_EXTRA_INIT \ .ob_refcnt = _Py_IMMORTAL_REFCNT, \ .ob_type = (type) \ - }, + } +#endif #define _PyVarObject_HEAD_INIT(type, size) \ { \ - .ob_base = _PyObject_HEAD_INIT(type) \ + .ob_base = _PyObject_HEAD_INIT(type), \ .ob_size = size \ - }, + } -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( +extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc( const char *func, const char *message); @@ -64,18 +103,73 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) #ifdef Py_REF_DEBUG _Py_AddRefTotal(_PyInterpreterState_GET(), n); #endif +#if !defined(Py_GIL_DISABLED) op->ob_refcnt += n; +#else + if (_Py_IsOwnedByCurrentThread(op)) { + uint32_t local = op->ob_ref_local; + Py_ssize_t refcnt = (Py_ssize_t)local + n; +# if PY_SSIZE_T_MAX > UINT32_MAX + if (refcnt > (Py_ssize_t)UINT32_MAX) { + // Make the object immortal if the 32-bit local reference count + // would overflow. + refcnt = _Py_IMMORTAL_REFCNT_LOCAL; + } +# endif + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, (uint32_t)refcnt); + } + else { + _Py_atomic_add_ssize(&op->ob_ref_shared, (n << _Py_REF_SHARED_SHIFT)); + } +#endif } #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) static inline void _Py_SetImmortal(PyObject *op) { if (op) { +#ifdef Py_GIL_DISABLED + op->ob_tid = _Py_UNOWNED_TID; + op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + op->ob_ref_shared = 0; +#else op->ob_refcnt = _Py_IMMORTAL_REFCNT; +#endif } } #define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) +// Makes an immortal object mortal again with the specified refcnt. Should only +// be used during runtime finalization. +static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt) +{ + if (op) { + assert(_Py_IsImmortal(op)); +#ifdef Py_GIL_DISABLED + op->ob_tid = _Py_UNOWNED_TID; + op->ob_ref_local = 0; + op->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); +#else + op->ob_refcnt = refcnt; +#endif + } +} + +/* _Py_ClearImmortal() should only be used during runtime finalization. */ +static inline void _Py_ClearImmortal(PyObject *op) +{ + if (op) { + _Py_SetMortal(op, 1); + Py_DECREF(op); + } +} +#define _Py_ClearImmortal(op) \ + do { \ + _Py_ClearImmortal(_PyObject_CAST(op)); \ + op = NULL; \ + } while (0) + +#if !defined(Py_GIL_DISABLED) static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { @@ -115,13 +209,44 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) #endif } +#else +// TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build +static inline void +_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) +{ + Py_DECREF(op); +} + +static inline void +_Py_DECREF_NO_DEALLOC(PyObject *op) +{ + Py_DECREF(op); +} + +static inline int +_Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) +{ + return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED; +} + +static inline int +_Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) +{ + return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; +} + +// Merge the local and shared reference count fields and add `extra` to the +// refcount when merging. +Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); +#endif // !defined(Py_GIL_DISABLED) + #ifdef Py_REF_DEBUG # undef _Py_DEC_REFTOTAL #endif -PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type); -PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); +extern int _PyType_CheckConsistency(PyTypeObject *type); +extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); /* Update the Python traceback of an object. This function must be called when a memory block is reused from a free list. @@ -137,6 +262,9 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { extern void _PyType_InitCache(PyInterpreterState *interp); +extern PyStatus _PyObject_InitState(PyInterpreterState *interp); +extern void _PyObject_FiniState(PyInterpreterState *interp); +extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and @@ -255,9 +383,9 @@ extern void _PyDebug_PrintTotalRefs(void); #endif #ifdef Py_TRACE_REFS -extern void _Py_AddToAllObjects(PyObject *op, int force); -extern void _Py_PrintReferences(FILE *); -extern void _Py_PrintReferenceAddresses(FILE *); +extern void _Py_AddToAllObjects(PyObject *op); +extern void _Py_PrintReferences(PyInterpreterState *, FILE *); +extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); #endif @@ -311,9 +439,9 @@ _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op) static inline int _PyObject_IS_GC(PyObject *obj) { - return (PyType_IS_GC(Py_TYPE(obj)) - && (Py_TYPE(obj)->tp_is_gc == NULL - || Py_TYPE(obj)->tp_is_gc(obj))); + PyTypeObject *type = Py_TYPE(obj); + return (PyType_IS_GC(type) + && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } // Fast inlined version of PyType_IS_GC() @@ -340,8 +468,14 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); +extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); + +extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); +extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); +extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); extern int _PyObject_InitializeDict(PyObject *obj); +int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value); PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, @@ -392,7 +526,17 @@ extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); -PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *); +// Export for 'math' shared extension +PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); + +extern int _PyObject_IsAbstract(PyObject *); + +extern int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +extern PyObject* _PyObject_NextNotImplemented(PyObject *); + +// Pickle support. +// Export for '_datetime' shared extension +PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *); /* C function call trampolines to mitigate bad function pointer casts. * @@ -421,6 +565,14 @@ extern PyObject* _PyCFunctionWithKeywords_TrampolineCall( (meth)((self), (args), (kw)) #endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE +// Export these 2 symbols for '_pickle' shared extension +PyAPI_DATA(PyTypeObject) _PyNone_Type; +PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; + +// Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. +// Export for the stable ABI. +PyAPI_DATA(int) _Py_SwappedOp[]; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 94005d77881432..9eac27b1a9a4e3 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -8,20 +8,26 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_hashtable.h" // _Py_hashtable_t + struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; -#else - int _not_used; #endif + int _not_used; }; struct _py_object_state { #ifdef Py_REF_DEBUG Py_ssize_t reftotal; -#else - int _not_used; #endif +#ifdef Py_TRACE_REFS + // Hash table storing all objects. The key is the object pointer + // (PyObject*) and the value is always the number 1 (as uintptr_t). + // See _PyRefchain_IsTraced() and _PyRefchain_Trace() functions. + _Py_hashtable_t *refchain; +#endif + int _not_used; }; diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index ca2a0419b4f038..b0dbf53d4e3d15 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -687,7 +687,7 @@ extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *); #ifdef WITH_PYMALLOC -// Export the symbol for the 3rd party guppy3 project +// Export the symbol for the 3rd party 'guppy3' project PyAPI_FUNC(int) _PyObject_DebugMallocStats(FILE *out); #endif diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h deleted file mode 100644 index e823e1bfb5d452..00000000000000 --- a/Include/internal/pycore_opcode.h +++ /dev/null @@ -1,587 +0,0 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py - -#ifndef Py_INTERNAL_OPCODE_H -#define Py_INTERNAL_OPCODE_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "opcode.h" - -extern const uint32_t _PyOpcode_Jump[9]; - -extern const uint8_t _PyOpcode_Caches[256]; - -extern const uint8_t _PyOpcode_Deopt[256]; - -#ifdef NEED_OPCODE_TABLES -const uint32_t _PyOpcode_Jump[9] = { - 0U, - 0U, - 536870912U, - 135020544U, - 4163U, - 0U, - 0U, - 0U, - 48U, -}; - -const uint8_t _PyOpcode_Caches[256] = { - [BINARY_SUBSCR] = 1, - [STORE_SUBSCR] = 1, - [UNPACK_SEQUENCE] = 1, - [FOR_ITER] = 1, - [STORE_ATTR] = 4, - [LOAD_ATTR] = 9, - [COMPARE_OP] = 1, - [LOAD_GLOBAL] = 4, - [BINARY_OP] = 1, - [SEND] = 1, - [LOAD_SUPER_ATTR] = 1, - [CALL] = 3, -}; - -const uint8_t _PyOpcode_Deopt[256] = { - [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, - [BEFORE_WITH] = BEFORE_WITH, - [BINARY_OP] = BINARY_OP, - [BINARY_OP_ADD_FLOAT] = BINARY_OP, - [BINARY_OP_ADD_INT] = BINARY_OP, - [BINARY_OP_ADD_UNICODE] = BINARY_OP, - [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, - [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, - [BINARY_OP_MULTIPLY_INT] = BINARY_OP, - [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, - [BINARY_OP_SUBTRACT_INT] = BINARY_OP, - [BINARY_SLICE] = BINARY_SLICE, - [BINARY_SUBSCR] = BINARY_SUBSCR, - [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, - [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, - [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, - [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, - [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, - [BUILD_LIST] = BUILD_LIST, - [BUILD_MAP] = BUILD_MAP, - [BUILD_SET] = BUILD_SET, - [BUILD_SLICE] = BUILD_SLICE, - [BUILD_STRING] = BUILD_STRING, - [BUILD_TUPLE] = BUILD_TUPLE, - [CACHE] = CACHE, - [CALL] = CALL, - [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, - [CALL_BUILTIN_CLASS] = CALL, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, - [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, - [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, - [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, - [CALL_NO_KW_BUILTIN_FAST] = CALL, - [CALL_NO_KW_BUILTIN_O] = CALL, - [CALL_NO_KW_ISINSTANCE] = CALL, - [CALL_NO_KW_LEN] = CALL, - [CALL_NO_KW_LIST_APPEND] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = CALL, - [CALL_NO_KW_STR_1] = CALL, - [CALL_NO_KW_TUPLE_1] = CALL, - [CALL_NO_KW_TYPE_1] = CALL, - [CALL_PY_EXACT_ARGS] = CALL, - [CALL_PY_WITH_DEFAULTS] = CALL, - [CHECK_EG_MATCH] = CHECK_EG_MATCH, - [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, - [CLEANUP_THROW] = CLEANUP_THROW, - [COMPARE_OP] = COMPARE_OP, - [COMPARE_OP_FLOAT] = COMPARE_OP, - [COMPARE_OP_INT] = COMPARE_OP, - [COMPARE_OP_STR] = COMPARE_OP, - [CONTAINS_OP] = CONTAINS_OP, - [COPY] = COPY, - [COPY_FREE_VARS] = COPY_FREE_VARS, - [DELETE_ATTR] = DELETE_ATTR, - [DELETE_DEREF] = DELETE_DEREF, - [DELETE_FAST] = DELETE_FAST, - [DELETE_GLOBAL] = DELETE_GLOBAL, - [DELETE_NAME] = DELETE_NAME, - [DELETE_SUBSCR] = DELETE_SUBSCR, - [DICT_MERGE] = DICT_MERGE, - [DICT_UPDATE] = DICT_UPDATE, - [END_ASYNC_FOR] = END_ASYNC_FOR, - [END_FOR] = END_FOR, - [END_SEND] = END_SEND, - [EXTENDED_ARG] = EXTENDED_ARG, - [FORMAT_VALUE] = FORMAT_VALUE, - [FOR_ITER] = FOR_ITER, - [FOR_ITER_GEN] = FOR_ITER, - [FOR_ITER_LIST] = FOR_ITER, - [FOR_ITER_RANGE] = FOR_ITER, - [FOR_ITER_TUPLE] = FOR_ITER, - [GET_AITER] = GET_AITER, - [GET_ANEXT] = GET_ANEXT, - [GET_AWAITABLE] = GET_AWAITABLE, - [GET_ITER] = GET_ITER, - [GET_LEN] = GET_LEN, - [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, - [IMPORT_FROM] = IMPORT_FROM, - [IMPORT_NAME] = IMPORT_NAME, - [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, - [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, - [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, - [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, - [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, - [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, - [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, - [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, - [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, - [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [INTERPRETER_EXIT] = INTERPRETER_EXIT, - [IS_OP] = IS_OP, - [JUMP_BACKWARD] = JUMP_BACKWARD, - [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, - [JUMP_FORWARD] = JUMP_FORWARD, - [KW_NAMES] = KW_NAMES, - [LIST_APPEND] = LIST_APPEND, - [LIST_EXTEND] = LIST_EXTEND, - [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, - [LOAD_ATTR] = LOAD_ATTR, - [LOAD_ATTR_CLASS] = LOAD_ATTR, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, - [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, - [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, - [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, - [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, - [LOAD_ATTR_MODULE] = LOAD_ATTR, - [LOAD_ATTR_PROPERTY] = LOAD_ATTR, - [LOAD_ATTR_SLOT] = LOAD_ATTR, - [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, - [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, - [LOAD_CLASSDEREF] = LOAD_CLASSDEREF, - [LOAD_CLOSURE] = LOAD_CLOSURE, - [LOAD_CONST] = LOAD_CONST, - [LOAD_CONST__LOAD_FAST] = LOAD_CONST, - [LOAD_DEREF] = LOAD_DEREF, - [LOAD_FAST] = LOAD_FAST, - [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, - [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, - [LOAD_FAST__LOAD_CONST] = LOAD_FAST, - [LOAD_FAST__LOAD_FAST] = LOAD_FAST, - [LOAD_GLOBAL] = LOAD_GLOBAL, - [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, - [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, - [LOAD_NAME] = LOAD_NAME, - [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, - [MAKE_CELL] = MAKE_CELL, - [MAKE_FUNCTION] = MAKE_FUNCTION, - [MAP_ADD] = MAP_ADD, - [MATCH_CLASS] = MATCH_CLASS, - [MATCH_KEYS] = MATCH_KEYS, - [MATCH_MAPPING] = MATCH_MAPPING, - [MATCH_SEQUENCE] = MATCH_SEQUENCE, - [NOP] = NOP, - [POP_EXCEPT] = POP_EXCEPT, - [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, - [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, - [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, - [POP_TOP] = POP_TOP, - [PUSH_EXC_INFO] = PUSH_EXC_INFO, - [PUSH_NULL] = PUSH_NULL, - [RAISE_VARARGS] = RAISE_VARARGS, - [RERAISE] = RERAISE, - [RESERVED] = RESERVED, - [RESUME] = RESUME, - [RETURN_CONST] = RETURN_CONST, - [RETURN_GENERATOR] = RETURN_GENERATOR, - [RETURN_VALUE] = RETURN_VALUE, - [SEND] = SEND, - [SEND_GEN] = SEND, - [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, - [SET_ADD] = SET_ADD, - [SET_UPDATE] = SET_UPDATE, - [STORE_ATTR] = STORE_ATTR, - [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, - [STORE_ATTR_SLOT] = STORE_ATTR, - [STORE_ATTR_WITH_HINT] = STORE_ATTR, - [STORE_DEREF] = STORE_DEREF, - [STORE_FAST] = STORE_FAST, - [STORE_FAST__LOAD_FAST] = STORE_FAST, - [STORE_FAST__STORE_FAST] = STORE_FAST, - [STORE_GLOBAL] = STORE_GLOBAL, - [STORE_NAME] = STORE_NAME, - [STORE_SLICE] = STORE_SLICE, - [STORE_SUBSCR] = STORE_SUBSCR, - [STORE_SUBSCR_DICT] = STORE_SUBSCR, - [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, - [SWAP] = SWAP, - [UNARY_INVERT] = UNARY_INVERT, - [UNARY_NEGATIVE] = UNARY_NEGATIVE, - [UNARY_NOT] = UNARY_NOT, - [UNPACK_EX] = UNPACK_EX, - [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, - [WITH_EXCEPT_START] = WITH_EXCEPT_START, - [YIELD_VALUE] = YIELD_VALUE, -}; -#endif // NEED_OPCODE_TABLES - -#ifdef Py_DEBUG -static const char *const _PyOpcode_OpName[267] = { - [CACHE] = "CACHE", - [POP_TOP] = "POP_TOP", - [PUSH_NULL] = "PUSH_NULL", - [INTERPRETER_EXIT] = "INTERPRETER_EXIT", - [END_FOR] = "END_FOR", - [END_SEND] = "END_SEND", - [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", - [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", - [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [NOP] = "NOP", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", - [UNARY_NEGATIVE] = "UNARY_NEGATIVE", - [UNARY_NOT] = "UNARY_NOT", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", - [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [UNARY_INVERT] = "UNARY_INVERT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", - [RESERVED] = "RESERVED", - [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", - [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", - [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", - [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", - [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [BINARY_SUBSCR] = "BINARY_SUBSCR", - [BINARY_SLICE] = "BINARY_SLICE", - [STORE_SLICE] = "STORE_SLICE", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [GET_LEN] = "GET_LEN", - [MATCH_MAPPING] = "MATCH_MAPPING", - [MATCH_SEQUENCE] = "MATCH_SEQUENCE", - [MATCH_KEYS] = "MATCH_KEYS", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [PUSH_EXC_INFO] = "PUSH_EXC_INFO", - [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", - [CHECK_EG_MATCH] = "CHECK_EG_MATCH", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", - [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", - [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", - [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", - [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [WITH_EXCEPT_START] = "WITH_EXCEPT_START", - [GET_AITER] = "GET_AITER", - [GET_ANEXT] = "GET_ANEXT", - [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", - [BEFORE_WITH] = "BEFORE_WITH", - [END_ASYNC_FOR] = "END_ASYNC_FOR", - [CLEANUP_THROW] = "CLEANUP_THROW", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", - [COMPARE_OP_INT] = "COMPARE_OP_INT", - [COMPARE_OP_STR] = "COMPARE_OP_STR", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [FOR_ITER_GEN] = "FOR_ITER_GEN", - [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", - [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [POP_EXCEPT] = "POP_EXCEPT", - [STORE_NAME] = "STORE_NAME", - [DELETE_NAME] = "DELETE_NAME", - [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", - [FOR_ITER] = "FOR_ITER", - [UNPACK_EX] = "UNPACK_EX", - [STORE_ATTR] = "STORE_ATTR", - [DELETE_ATTR] = "DELETE_ATTR", - [STORE_GLOBAL] = "STORE_GLOBAL", - [DELETE_GLOBAL] = "DELETE_GLOBAL", - [SWAP] = "SWAP", - [LOAD_CONST] = "LOAD_CONST", - [LOAD_NAME] = "LOAD_NAME", - [BUILD_TUPLE] = "BUILD_TUPLE", - [BUILD_LIST] = "BUILD_LIST", - [BUILD_SET] = "BUILD_SET", - [BUILD_MAP] = "BUILD_MAP", - [LOAD_ATTR] = "LOAD_ATTR", - [COMPARE_OP] = "COMPARE_OP", - [IMPORT_NAME] = "IMPORT_NAME", - [IMPORT_FROM] = "IMPORT_FROM", - [JUMP_FORWARD] = "JUMP_FORWARD", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", - [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", - [LOAD_GLOBAL] = "LOAD_GLOBAL", - [IS_OP] = "IS_OP", - [CONTAINS_OP] = "CONTAINS_OP", - [RERAISE] = "RERAISE", - [COPY] = "COPY", - [RETURN_CONST] = "RETURN_CONST", - [BINARY_OP] = "BINARY_OP", - [SEND] = "SEND", - [LOAD_FAST] = "LOAD_FAST", - [STORE_FAST] = "STORE_FAST", - [DELETE_FAST] = "DELETE_FAST", - [LOAD_FAST_CHECK] = "LOAD_FAST_CHECK", - [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", - [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", - [RAISE_VARARGS] = "RAISE_VARARGS", - [GET_AWAITABLE] = "GET_AWAITABLE", - [MAKE_FUNCTION] = "MAKE_FUNCTION", - [BUILD_SLICE] = "BUILD_SLICE", - [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", - [MAKE_CELL] = "MAKE_CELL", - [LOAD_CLOSURE] = "LOAD_CLOSURE", - [LOAD_DEREF] = "LOAD_DEREF", - [STORE_DEREF] = "STORE_DEREF", - [DELETE_DEREF] = "DELETE_DEREF", - [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", - [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", - [EXTENDED_ARG] = "EXTENDED_ARG", - [LIST_APPEND] = "LIST_APPEND", - [SET_ADD] = "SET_ADD", - [MAP_ADD] = "MAP_ADD", - [LOAD_CLASSDEREF] = "LOAD_CLASSDEREF", - [COPY_FREE_VARS] = "COPY_FREE_VARS", - [YIELD_VALUE] = "YIELD_VALUE", - [RESUME] = "RESUME", - [MATCH_CLASS] = "MATCH_CLASS", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [FORMAT_VALUE] = "FORMAT_VALUE", - [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", - [BUILD_STRING] = "BUILD_STRING", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", - [LIST_EXTEND] = "LIST_EXTEND", - [SET_UPDATE] = "SET_UPDATE", - [DICT_MERGE] = "DICT_MERGE", - [DICT_UPDATE] = "DICT_UPDATE", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [SEND_GEN] = "SEND_GEN", - [169] = "<169>", - [170] = "<170>", - [CALL] = "CALL", - [KW_NAMES] = "KW_NAMES", - [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", - [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", - [175] = "<175>", - [176] = "<176>", - [177] = "<177>", - [178] = "<178>", - [179] = "<179>", - [180] = "<180>", - [181] = "<181>", - [182] = "<182>", - [183] = "<183>", - [184] = "<184>", - [185] = "<185>", - [186] = "<186>", - [187] = "<187>", - [188] = "<188>", - [189] = "<189>", - [190] = "<190>", - [191] = "<191>", - [192] = "<192>", - [193] = "<193>", - [194] = "<194>", - [195] = "<195>", - [196] = "<196>", - [197] = "<197>", - [198] = "<198>", - [199] = "<199>", - [200] = "<200>", - [201] = "<201>", - [202] = "<202>", - [203] = "<203>", - [204] = "<204>", - [205] = "<205>", - [206] = "<206>", - [207] = "<207>", - [208] = "<208>", - [209] = "<209>", - [210] = "<210>", - [211] = "<211>", - [212] = "<212>", - [213] = "<213>", - [214] = "<214>", - [215] = "<215>", - [216] = "<216>", - [217] = "<217>", - [218] = "<218>", - [219] = "<219>", - [220] = "<220>", - [221] = "<221>", - [222] = "<222>", - [223] = "<223>", - [224] = "<224>", - [225] = "<225>", - [226] = "<226>", - [227] = "<227>", - [228] = "<228>", - [229] = "<229>", - [230] = "<230>", - [231] = "<231>", - [232] = "<232>", - [233] = "<233>", - [234] = "<234>", - [235] = "<235>", - [236] = "<236>", - [237] = "<237>", - [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", - [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", - [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", - [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", - [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", - [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", - [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", - [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", - [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", - [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", - [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", - [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", - [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", - [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [255] = "<255>", - [SETUP_FINALLY] = "SETUP_FINALLY", - [SETUP_CLEANUP] = "SETUP_CLEANUP", - [SETUP_WITH] = "SETUP_WITH", - [POP_BLOCK] = "POP_BLOCK", - [JUMP] = "JUMP", - [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", - [LOAD_METHOD] = "LOAD_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", -}; -#endif - -#define EXTRA_CASES \ - case 169: \ - case 170: \ - case 175: \ - case 176: \ - case 177: \ - case 178: \ - case 179: \ - case 180: \ - case 181: \ - case 182: \ - case 183: \ - case 184: \ - case 185: \ - case 186: \ - case 187: \ - case 188: \ - case 189: \ - case 190: \ - case 191: \ - case 192: \ - case 193: \ - case 194: \ - case 195: \ - case 196: \ - case 197: \ - case 198: \ - case 199: \ - case 200: \ - case 201: \ - case 202: \ - case 203: \ - case 204: \ - case 205: \ - case 206: \ - case 207: \ - case 208: \ - case 209: \ - case 210: \ - case 211: \ - case 212: \ - case 213: \ - case 214: \ - case 215: \ - case 216: \ - case 217: \ - case 218: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ - case 223: \ - case 224: \ - case 225: \ - case 226: \ - case 227: \ - case 228: \ - case 229: \ - case 230: \ - case 231: \ - case 232: \ - case 233: \ - case 234: \ - case 235: \ - case 236: \ - case 237: \ - case 255: \ - ; - -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_OPCODE_H diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h new file mode 100644 index 00000000000000..774c0f99379ed6 --- /dev/null +++ b/Include/internal/pycore_opcode_metadata.h @@ -0,0 +1,2573 @@ +// This file is generated by Tools/cases_generator/generate_cases.py +// from: +// Python/bytecodes.c +// Do not edit! + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include // bool + + +#define IS_PSEUDO_INSTR(OP) ( \ + ((OP) == LOAD_CLOSURE) || \ + ((OP) == STORE_FAST_MAYBE_NULL) || \ + ((OP) == LOAD_SUPER_METHOD) || \ + ((OP) == LOAD_ZERO_SUPER_METHOD) || \ + ((OP) == LOAD_ZERO_SUPER_ATTR) || \ + ((OP) == LOAD_METHOD) || \ + ((OP) == JUMP) || \ + ((OP) == JUMP_NO_INTERRUPT) || \ + ((OP) == SETUP_FINALLY) || \ + ((OP) == SETUP_CLEANUP) || \ + ((OP) == SETUP_WITH) || \ + ((OP) == POP_BLOCK) || \ + 0) + +#define _EXIT_TRACE 300 +#define _SET_IP 301 +#define _SPECIALIZE_TO_BOOL 302 +#define _TO_BOOL 303 +#define _GUARD_BOTH_INT 304 +#define _BINARY_OP_MULTIPLY_INT 305 +#define _BINARY_OP_ADD_INT 306 +#define _BINARY_OP_SUBTRACT_INT 307 +#define _GUARD_BOTH_FLOAT 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_ADD_FLOAT 310 +#define _BINARY_OP_SUBTRACT_FLOAT 311 +#define _GUARD_BOTH_UNICODE 312 +#define _BINARY_OP_ADD_UNICODE 313 +#define _BINARY_OP_INPLACE_ADD_UNICODE 314 +#define _SPECIALIZE_BINARY_SUBSCR 315 +#define _BINARY_SUBSCR 316 +#define _SPECIALIZE_STORE_SUBSCR 317 +#define _STORE_SUBSCR 318 +#define _POP_FRAME 319 +#define _SPECIALIZE_SEND 320 +#define _SEND 321 +#define _SPECIALIZE_UNPACK_SEQUENCE 322 +#define _UNPACK_SEQUENCE 323 +#define _SPECIALIZE_STORE_ATTR 324 +#define _STORE_ATTR 325 +#define _SPECIALIZE_LOAD_GLOBAL 326 +#define _LOAD_GLOBAL 327 +#define _GUARD_GLOBALS_VERSION 328 +#define _GUARD_BUILTINS_VERSION 329 +#define _LOAD_GLOBAL_MODULE 330 +#define _LOAD_GLOBAL_BUILTINS 331 +#define _SPECIALIZE_LOAD_SUPER_ATTR 332 +#define _LOAD_SUPER_ATTR 333 +#define _SPECIALIZE_LOAD_ATTR 334 +#define _LOAD_ATTR 335 +#define _GUARD_TYPE_VERSION 336 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 +#define _LOAD_ATTR_INSTANCE_VALUE 338 +#define _CHECK_ATTR_MODULE 339 +#define _LOAD_ATTR_MODULE 340 +#define _CHECK_ATTR_WITH_HINT 341 +#define _LOAD_ATTR_WITH_HINT 342 +#define _LOAD_ATTR_SLOT 343 +#define _CHECK_ATTR_CLASS 344 +#define _LOAD_ATTR_CLASS 345 +#define _GUARD_DORV_VALUES 346 +#define _STORE_ATTR_INSTANCE_VALUE 347 +#define _STORE_ATTR_SLOT 348 +#define _SPECIALIZE_COMPARE_OP 349 +#define _COMPARE_OP 350 +#define _POP_JUMP_IF_FALSE 351 +#define _POP_JUMP_IF_TRUE 352 +#define _IS_NONE 353 +#define _SPECIALIZE_FOR_ITER 354 +#define _FOR_ITER 355 +#define _FOR_ITER_TIER_TWO 356 +#define _ITER_CHECK_LIST 357 +#define _ITER_JUMP_LIST 358 +#define _GUARD_NOT_EXHAUSTED_LIST 359 +#define _ITER_NEXT_LIST 360 +#define _ITER_CHECK_TUPLE 361 +#define _ITER_JUMP_TUPLE 362 +#define _GUARD_NOT_EXHAUSTED_TUPLE 363 +#define _ITER_NEXT_TUPLE 364 +#define _ITER_CHECK_RANGE 365 +#define _ITER_JUMP_RANGE 366 +#define _GUARD_NOT_EXHAUSTED_RANGE 367 +#define _ITER_NEXT_RANGE 368 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 369 +#define _GUARD_KEYS_VERSION 370 +#define _LOAD_ATTR_METHOD_WITH_VALUES 371 +#define _LOAD_ATTR_METHOD_NO_DICT 372 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 373 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 374 +#define _CHECK_ATTR_METHOD_LAZY_DICT 375 +#define _LOAD_ATTR_METHOD_LAZY_DICT 376 +#define _SPECIALIZE_CALL 377 +#define _CALL 378 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 379 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 380 +#define _CHECK_PEP_523 381 +#define _CHECK_FUNCTION_EXACT_ARGS 382 +#define _CHECK_STACK_SPACE 383 +#define _INIT_CALL_PY_EXACT_ARGS 384 +#define _PUSH_FRAME 385 +#define _SPECIALIZE_BINARY_OP 386 +#define _BINARY_OP 387 +#define _GUARD_IS_TRUE_POP 388 +#define _GUARD_IS_FALSE_POP 389 +#define _GUARD_IS_NONE_POP 390 +#define _GUARD_IS_NOT_NONE_POP 391 +#define _JUMP_TO_TOP 392 +#define _SAVE_RETURN_OFFSET 393 +#define _INSERT 394 +#define _CHECK_VALIDITY 395 + +extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { + switch(opcode) { + case NOP: + return 0; + case RESUME: + return 0; + case RESUME_CHECK: + return 0; + case INSTRUMENTED_RESUME: + return 0; + case LOAD_CLOSURE: + return 0; + case LOAD_FAST_CHECK: + return 0; + case LOAD_FAST: + return 0; + case LOAD_FAST_AND_CLEAR: + return 0; + case LOAD_FAST_LOAD_FAST: + return 0; + case LOAD_CONST: + return 0; + case STORE_FAST: + return 1; + case STORE_FAST_MAYBE_NULL: + return 1; + case STORE_FAST_LOAD_FAST: + return 1; + case STORE_FAST_STORE_FAST: + return 2; + case POP_TOP: + return 1; + case PUSH_NULL: + return 0; + case END_FOR: + return 2; + case INSTRUMENTED_END_FOR: + return 2; + case END_SEND: + return 2; + case INSTRUMENTED_END_SEND: + return 2; + case UNARY_NEGATIVE: + return 1; + case UNARY_NOT: + return 1; + case _SPECIALIZE_TO_BOOL: + return 1; + case _TO_BOOL: + return 1; + case TO_BOOL: + return 1; + case TO_BOOL_BOOL: + return 1; + case TO_BOOL_INT: + return 1; + case TO_BOOL_LIST: + return 1; + case TO_BOOL_NONE: + return 1; + case TO_BOOL_STR: + return 1; + case TO_BOOL_ALWAYS_TRUE: + return 1; + case UNARY_INVERT: + return 1; + case _GUARD_BOTH_INT: + return 2; + case _BINARY_OP_MULTIPLY_INT: + return 2; + case _BINARY_OP_ADD_INT: + return 2; + case _BINARY_OP_SUBTRACT_INT: + return 2; + case BINARY_OP_MULTIPLY_INT: + return 2; + case BINARY_OP_ADD_INT: + return 2; + case BINARY_OP_SUBTRACT_INT: + return 2; + case _GUARD_BOTH_FLOAT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT: + return 2; + case _BINARY_OP_ADD_FLOAT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT: + return 2; + case BINARY_OP_MULTIPLY_FLOAT: + return 2; + case BINARY_OP_ADD_FLOAT: + return 2; + case BINARY_OP_SUBTRACT_FLOAT: + return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 2; + case BINARY_OP_ADD_UNICODE: + return 2; + case _BINARY_OP_INPLACE_ADD_UNICODE: + return 2; + case BINARY_OP_INPLACE_ADD_UNICODE: + return 2; + case _SPECIALIZE_BINARY_SUBSCR: + return 2; + case _BINARY_SUBSCR: + return 2; + case BINARY_SUBSCR: + return 2; + case BINARY_SLICE: + return 3; + case STORE_SLICE: + return 4; + case BINARY_SUBSCR_LIST_INT: + return 2; + case BINARY_SUBSCR_STR_INT: + return 2; + case BINARY_SUBSCR_TUPLE_INT: + return 2; + case BINARY_SUBSCR_DICT: + return 2; + case BINARY_SUBSCR_GETITEM: + return 2; + case LIST_APPEND: + return (oparg-1) + 2; + case SET_ADD: + return (oparg-1) + 2; + case _SPECIALIZE_STORE_SUBSCR: + return 2; + case _STORE_SUBSCR: + return 3; + case STORE_SUBSCR: + return 3; + case STORE_SUBSCR_LIST_INT: + return 3; + case STORE_SUBSCR_DICT: + return 3; + case DELETE_SUBSCR: + return 2; + case CALL_INTRINSIC_1: + return 1; + case CALL_INTRINSIC_2: + return 2; + case RAISE_VARARGS: + return oparg; + case INTERPRETER_EXIT: + return 1; + case _POP_FRAME: + return 1; + case RETURN_VALUE: + return 1; + case INSTRUMENTED_RETURN_VALUE: + return 1; + case RETURN_CONST: + return 0; + case INSTRUMENTED_RETURN_CONST: + return 0; + case GET_AITER: + return 1; + case GET_ANEXT: + return 1; + case GET_AWAITABLE: + return 1; + case _SPECIALIZE_SEND: + return 2; + case _SEND: + return 2; + case SEND: + return 2; + case SEND_GEN: + return 2; + case INSTRUMENTED_YIELD_VALUE: + return 1; + case YIELD_VALUE: + return 1; + case POP_EXCEPT: + return 1; + case RERAISE: + return oparg + 1; + case END_ASYNC_FOR: + return 2; + case CLEANUP_THROW: + return 3; + case LOAD_ASSERTION_ERROR: + return 0; + case LOAD_BUILD_CLASS: + return 0; + case STORE_NAME: + return 1; + case DELETE_NAME: + return 0; + case _SPECIALIZE_UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE: + return 1; + case UNPACK_SEQUENCE: + return 1; + case UNPACK_SEQUENCE_TWO_TUPLE: + return 1; + case UNPACK_SEQUENCE_TUPLE: + return 1; + case UNPACK_SEQUENCE_LIST: + return 1; + case UNPACK_EX: + return 1; + case _SPECIALIZE_STORE_ATTR: + return 1; + case _STORE_ATTR: + return 2; + case STORE_ATTR: + return 2; + case DELETE_ATTR: + return 1; + case STORE_GLOBAL: + return 1; + case DELETE_GLOBAL: + return 0; + case LOAD_LOCALS: + return 0; + case LOAD_FROM_DICT_OR_GLOBALS: + return 1; + case LOAD_NAME: + return 0; + case _SPECIALIZE_LOAD_GLOBAL: + return 0; + case _LOAD_GLOBAL: + return 0; + case LOAD_GLOBAL: + return 0; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; + case LOAD_GLOBAL_MODULE: + return 0; + case LOAD_GLOBAL_BUILTIN: + return 0; + case DELETE_FAST: + return 0; + case MAKE_CELL: + return 0; + case DELETE_DEREF: + return 0; + case LOAD_FROM_DICT_OR_DEREF: + return 1; + case LOAD_DEREF: + return 0; + case STORE_DEREF: + return 1; + case COPY_FREE_VARS: + return 0; + case BUILD_STRING: + return oparg; + case BUILD_TUPLE: + return oparg; + case BUILD_LIST: + return oparg; + case LIST_EXTEND: + return (oparg-1) + 2; + case SET_UPDATE: + return (oparg-1) + 2; + case BUILD_SET: + return oparg; + case BUILD_MAP: + return oparg*2; + case SETUP_ANNOTATIONS: + return 0; + case BUILD_CONST_KEY_MAP: + return oparg + 1; + case DICT_UPDATE: + return (oparg - 1) + 2; + case DICT_MERGE: + return (oparg - 1) + 5; + case MAP_ADD: + return (oparg - 1) + 3; + case INSTRUMENTED_LOAD_SUPER_ATTR: + return 3; + case _SPECIALIZE_LOAD_SUPER_ATTR: + return 3; + case _LOAD_SUPER_ATTR: + return 3; + case LOAD_SUPER_ATTR: + return 3; + case LOAD_SUPER_METHOD: + return 3; + case LOAD_ZERO_SUPER_METHOD: + return 3; + case LOAD_ZERO_SUPER_ATTR: + return 3; + case LOAD_SUPER_ATTR_ATTR: + return 3; + case LOAD_SUPER_ATTR_METHOD: + return 3; + case _SPECIALIZE_LOAD_ATTR: + return 1; + case _LOAD_ATTR: + return 1; + case LOAD_ATTR: + return 1; + case LOAD_METHOD: + return 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return 1; + case LOAD_ATTR_INSTANCE_VALUE: + return 1; + case _CHECK_ATTR_MODULE: + return 1; + case _LOAD_ATTR_MODULE: + return 1; + case LOAD_ATTR_MODULE: + return 1; + case _CHECK_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_WITH_HINT: + return 1; + case LOAD_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_SLOT: + return 1; + case LOAD_ATTR_SLOT: + return 1; + case _CHECK_ATTR_CLASS: + return 1; + case _LOAD_ATTR_CLASS: + return 1; + case LOAD_ATTR_CLASS: + return 1; + case LOAD_ATTR_PROPERTY: + return 1; + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + return 1; + case _GUARD_DORV_VALUES: + return 1; + case _STORE_ATTR_INSTANCE_VALUE: + return 2; + case STORE_ATTR_INSTANCE_VALUE: + return 2; + case STORE_ATTR_WITH_HINT: + return 2; + case _STORE_ATTR_SLOT: + return 2; + case STORE_ATTR_SLOT: + return 2; + case _SPECIALIZE_COMPARE_OP: + return 2; + case _COMPARE_OP: + return 2; + case COMPARE_OP: + return 2; + case COMPARE_OP_FLOAT: + return 2; + case COMPARE_OP_INT: + return 2; + case COMPARE_OP_STR: + return 2; + case IS_OP: + return 2; + case CONTAINS_OP: + return 2; + case CHECK_EG_MATCH: + return 2; + case CHECK_EXC_MATCH: + return 2; + case IMPORT_NAME: + return 2; + case IMPORT_FROM: + return 1; + case JUMP_FORWARD: + return 0; + case JUMP_BACKWARD: + return 0; + case JUMP: + return 0; + case JUMP_NO_INTERRUPT: + return 0; + case ENTER_EXECUTOR: + return 0; + case _POP_JUMP_IF_FALSE: + return 1; + case _POP_JUMP_IF_TRUE: + return 1; + case _IS_NONE: + return 1; + case POP_JUMP_IF_TRUE: + return 1; + case POP_JUMP_IF_FALSE: + return 1; + case POP_JUMP_IF_NONE: + return 1; + case POP_JUMP_IF_NOT_NONE: + return 1; + case JUMP_BACKWARD_NO_INTERRUPT: + return 0; + case GET_LEN: + return 1; + case MATCH_CLASS: + return 3; + case MATCH_MAPPING: + return 1; + case MATCH_SEQUENCE: + return 1; + case MATCH_KEYS: + return 2; + case GET_ITER: + return 1; + case GET_YIELD_FROM_ITER: + return 1; + case _SPECIALIZE_FOR_ITER: + return 1; + case _FOR_ITER: + return 1; + case _FOR_ITER_TIER_TWO: + return 1; + case FOR_ITER: + return 1; + case INSTRUMENTED_FOR_ITER: + return 0; + case _ITER_CHECK_LIST: + return 1; + case _ITER_JUMP_LIST: + return 1; + case _GUARD_NOT_EXHAUSTED_LIST: + return 1; + case _ITER_NEXT_LIST: + return 1; + case FOR_ITER_LIST: + return 1; + case _ITER_CHECK_TUPLE: + return 1; + case _ITER_JUMP_TUPLE: + return 1; + case _GUARD_NOT_EXHAUSTED_TUPLE: + return 1; + case _ITER_NEXT_TUPLE: + return 1; + case FOR_ITER_TUPLE: + return 1; + case _ITER_CHECK_RANGE: + return 1; + case _ITER_JUMP_RANGE: + return 1; + case _GUARD_NOT_EXHAUSTED_RANGE: + return 1; + case _ITER_NEXT_RANGE: + return 1; + case FOR_ITER_RANGE: + return 1; + case FOR_ITER_GEN: + return 1; + case BEFORE_ASYNC_WITH: + return 1; + case BEFORE_WITH: + return 1; + case WITH_EXCEPT_START: + return 4; + case SETUP_FINALLY: + return 0; + case SETUP_CLEANUP: + return 0; + case SETUP_WITH: + return 0; + case POP_BLOCK: + return 0; + case PUSH_EXC_INFO: + return 1; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_KEYS_VERSION: + return 1; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 1; + case LOAD_ATTR_METHOD_WITH_VALUES: + return 1; + case _LOAD_ATTR_METHOD_NO_DICT: + return 1; + case LOAD_ATTR_METHOD_NO_DICT: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 1; + case LOAD_ATTR_METHOD_LAZY_DICT: + return 1; + case INSTRUMENTED_CALL: + return 0; + case _SPECIALIZE_CALL: + return oparg + 2; + case _CALL: + return oparg + 2; + case CALL: + return oparg + 2; + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return oparg + 2; + case _CHECK_STACK_SPACE: + return oparg + 2; + case _INIT_CALL_PY_EXACT_ARGS: + return oparg + 2; + case _PUSH_FRAME: + return 1; + case CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case CALL_PY_EXACT_ARGS: + return oparg + 2; + case CALL_PY_WITH_DEFAULTS: + return oparg + 2; + case CALL_TYPE_1: + return oparg + 2; + case CALL_STR_1: + return oparg + 2; + case CALL_TUPLE_1: + return oparg + 2; + case CALL_ALLOC_AND_ENTER_INIT: + return oparg + 2; + case EXIT_INIT_CHECK: + return 1; + case CALL_BUILTIN_CLASS: + return oparg + 2; + case CALL_BUILTIN_O: + return oparg + 2; + case CALL_BUILTIN_FAST: + return oparg + 2; + case CALL_BUILTIN_FAST_WITH_KEYWORDS: + return oparg + 2; + case CALL_LEN: + return oparg + 2; + case CALL_ISINSTANCE: + return oparg + 2; + case CALL_LIST_APPEND: + return oparg + 2; + case CALL_METHOD_DESCRIPTOR_O: + return oparg + 2; + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return oparg + 2; + case CALL_METHOD_DESCRIPTOR_NOARGS: + return oparg + 2; + case CALL_METHOD_DESCRIPTOR_FAST: + return oparg + 2; + case INSTRUMENTED_CALL_KW: + return 0; + case CALL_KW: + return oparg + 3; + case INSTRUMENTED_CALL_FUNCTION_EX: + return 0; + case CALL_FUNCTION_EX: + return ((oparg & 1) ? 1 : 0) + 3; + case MAKE_FUNCTION: + return 1; + case SET_FUNCTION_ATTRIBUTE: + return 2; + case RETURN_GENERATOR: + return 0; + case BUILD_SLICE: + return ((oparg == 3) ? 1 : 0) + 2; + case CONVERT_VALUE: + return 1; + case FORMAT_SIMPLE: + return 1; + case FORMAT_WITH_SPEC: + return 2; + case COPY: + return (oparg-1) + 1; + case _SPECIALIZE_BINARY_OP: + return 2; + case _BINARY_OP: + return 2; + case BINARY_OP: + return 2; + case SWAP: + return (oparg-2) + 2; + case INSTRUMENTED_INSTRUCTION: + return 0; + case INSTRUMENTED_JUMP_FORWARD: + return 0; + case INSTRUMENTED_JUMP_BACKWARD: + return 0; + case INSTRUMENTED_POP_JUMP_IF_TRUE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_FALSE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NONE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: + return 0; + case EXTENDED_ARG: + return 0; + case CACHE: + return 0; + case RESERVED: + return 0; + case _GUARD_IS_TRUE_POP: + return 1; + case _GUARD_IS_FALSE_POP: + return 1; + case _GUARD_IS_NONE_POP: + return 1; + case _GUARD_IS_NOT_NONE_POP: + return 1; + case _JUMP_TO_TOP: + return 0; + case _SET_IP: + return 0; + case _SAVE_RETURN_OFFSET: + return 0; + case _EXIT_TRACE: + return 0; + case _INSERT: + return oparg + 1; + case _CHECK_VALIDITY: + return 0; + default: + return -1; + } +} +#endif // NEED_OPCODE_METADATA + +extern int _PyOpcode_num_pushed(int opcode, int oparg, bool jump); +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { + switch(opcode) { + case NOP: + return 0; + case RESUME: + return 0; + case RESUME_CHECK: + return 0; + case INSTRUMENTED_RESUME: + return 0; + case LOAD_CLOSURE: + return 1; + case LOAD_FAST_CHECK: + return 1; + case LOAD_FAST: + return 1; + case LOAD_FAST_AND_CLEAR: + return 1; + case LOAD_FAST_LOAD_FAST: + return 2; + case LOAD_CONST: + return 1; + case STORE_FAST: + return 0; + case STORE_FAST_MAYBE_NULL: + return 0; + case STORE_FAST_LOAD_FAST: + return 1; + case STORE_FAST_STORE_FAST: + return 0; + case POP_TOP: + return 0; + case PUSH_NULL: + return 1; + case END_FOR: + return 0; + case INSTRUMENTED_END_FOR: + return 0; + case END_SEND: + return 1; + case INSTRUMENTED_END_SEND: + return 1; + case UNARY_NEGATIVE: + return 1; + case UNARY_NOT: + return 1; + case _SPECIALIZE_TO_BOOL: + return 1; + case _TO_BOOL: + return 1; + case TO_BOOL: + return 1; + case TO_BOOL_BOOL: + return 1; + case TO_BOOL_INT: + return 1; + case TO_BOOL_LIST: + return 1; + case TO_BOOL_NONE: + return 1; + case TO_BOOL_STR: + return 1; + case TO_BOOL_ALWAYS_TRUE: + return 1; + case UNARY_INVERT: + return 1; + case _GUARD_BOTH_INT: + return 2; + case _BINARY_OP_MULTIPLY_INT: + return 1; + case _BINARY_OP_ADD_INT: + return 1; + case _BINARY_OP_SUBTRACT_INT: + return 1; + case BINARY_OP_MULTIPLY_INT: + return 1; + case BINARY_OP_ADD_INT: + return 1; + case BINARY_OP_SUBTRACT_INT: + return 1; + case _GUARD_BOTH_FLOAT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT: + return 1; + case _BINARY_OP_ADD_FLOAT: + return 1; + case _BINARY_OP_SUBTRACT_FLOAT: + return 1; + case BINARY_OP_MULTIPLY_FLOAT: + return 1; + case BINARY_OP_ADD_FLOAT: + return 1; + case BINARY_OP_SUBTRACT_FLOAT: + return 1; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 1; + case BINARY_OP_ADD_UNICODE: + return 1; + case _BINARY_OP_INPLACE_ADD_UNICODE: + return 0; + case BINARY_OP_INPLACE_ADD_UNICODE: + return 0; + case _SPECIALIZE_BINARY_SUBSCR: + return 2; + case _BINARY_SUBSCR: + return 1; + case BINARY_SUBSCR: + return 1; + case BINARY_SLICE: + return 1; + case STORE_SLICE: + return 0; + case BINARY_SUBSCR_LIST_INT: + return 1; + case BINARY_SUBSCR_STR_INT: + return 1; + case BINARY_SUBSCR_TUPLE_INT: + return 1; + case BINARY_SUBSCR_DICT: + return 1; + case BINARY_SUBSCR_GETITEM: + return 1; + case LIST_APPEND: + return (oparg-1) + 1; + case SET_ADD: + return (oparg-1) + 1; + case _SPECIALIZE_STORE_SUBSCR: + return 2; + case _STORE_SUBSCR: + return 0; + case STORE_SUBSCR: + return 0; + case STORE_SUBSCR_LIST_INT: + return 0; + case STORE_SUBSCR_DICT: + return 0; + case DELETE_SUBSCR: + return 0; + case CALL_INTRINSIC_1: + return 1; + case CALL_INTRINSIC_2: + return 1; + case RAISE_VARARGS: + return 0; + case INTERPRETER_EXIT: + return 0; + case _POP_FRAME: + return 0; + case RETURN_VALUE: + return 0; + case INSTRUMENTED_RETURN_VALUE: + return 0; + case RETURN_CONST: + return 0; + case INSTRUMENTED_RETURN_CONST: + return 0; + case GET_AITER: + return 1; + case GET_ANEXT: + return 2; + case GET_AWAITABLE: + return 1; + case _SPECIALIZE_SEND: + return 2; + case _SEND: + return 2; + case SEND: + return 2; + case SEND_GEN: + return 2; + case INSTRUMENTED_YIELD_VALUE: + return 1; + case YIELD_VALUE: + return 1; + case POP_EXCEPT: + return 0; + case RERAISE: + return oparg; + case END_ASYNC_FOR: + return 0; + case CLEANUP_THROW: + return 2; + case LOAD_ASSERTION_ERROR: + return 1; + case LOAD_BUILD_CLASS: + return 1; + case STORE_NAME: + return 0; + case DELETE_NAME: + return 0; + case _SPECIALIZE_UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE: + return oparg; + case UNPACK_SEQUENCE: + return oparg; + case UNPACK_SEQUENCE_TWO_TUPLE: + return oparg; + case UNPACK_SEQUENCE_TUPLE: + return oparg; + case UNPACK_SEQUENCE_LIST: + return oparg; + case UNPACK_EX: + return (oparg & 0xFF) + (oparg >> 8) + 1; + case _SPECIALIZE_STORE_ATTR: + return 1; + case _STORE_ATTR: + return 0; + case STORE_ATTR: + return 0; + case DELETE_ATTR: + return 0; + case STORE_GLOBAL: + return 0; + case DELETE_GLOBAL: + return 0; + case LOAD_LOCALS: + return 1; + case LOAD_FROM_DICT_OR_GLOBALS: + return 1; + case LOAD_NAME: + return 1; + case _SPECIALIZE_LOAD_GLOBAL: + return 0; + case _LOAD_GLOBAL: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_GLOBAL: + return (oparg & 1 ? 1 : 0) + 1; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_GLOBAL_BUILTINS: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_GLOBAL_MODULE: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_GLOBAL_BUILTIN: + return (oparg & 1 ? 1 : 0) + 1; + case DELETE_FAST: + return 0; + case MAKE_CELL: + return 0; + case DELETE_DEREF: + return 0; + case LOAD_FROM_DICT_OR_DEREF: + return 1; + case LOAD_DEREF: + return 1; + case STORE_DEREF: + return 0; + case COPY_FREE_VARS: + return 0; + case BUILD_STRING: + return 1; + case BUILD_TUPLE: + return 1; + case BUILD_LIST: + return 1; + case LIST_EXTEND: + return (oparg-1) + 1; + case SET_UPDATE: + return (oparg-1) + 1; + case BUILD_SET: + return 1; + case BUILD_MAP: + return 1; + case SETUP_ANNOTATIONS: + return 0; + case BUILD_CONST_KEY_MAP: + return 1; + case DICT_UPDATE: + return (oparg - 1) + 1; + case DICT_MERGE: + return (oparg - 1) + 4; + case MAP_ADD: + return (oparg - 1) + 1; + case INSTRUMENTED_LOAD_SUPER_ATTR: + return ((oparg & 1) ? 1 : 0) + 1; + case _SPECIALIZE_LOAD_SUPER_ATTR: + return 3; + case _LOAD_SUPER_ATTR: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_SUPER_ATTR: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_SUPER_METHOD: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_ZERO_SUPER_METHOD: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_ZERO_SUPER_ATTR: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_SUPER_ATTR_ATTR: + return 1; + case LOAD_SUPER_ATTR_METHOD: + return 2; + case _SPECIALIZE_LOAD_ATTR: + return 1; + case _LOAD_ATTR: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_METHOD: + return (oparg & 1 ? 1 : 0) + 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR_INSTANCE_VALUE: + return (oparg & 1 ? 1 : 0) + 1; + case _CHECK_ATTR_MODULE: + return 1; + case _LOAD_ATTR_MODULE: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR_MODULE: + return (oparg & 1 ? 1 : 0) + 1; + case _CHECK_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_WITH_HINT: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR_WITH_HINT: + return (oparg & 1 ? 1 : 0) + 1; + case _LOAD_ATTR_SLOT: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR_SLOT: + return (oparg & 1 ? 1 : 0) + 1; + case _CHECK_ATTR_CLASS: + return 1; + case _LOAD_ATTR_CLASS: + return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR_CLASS: + return (oparg & 1 ? 1 : 0) + 1; + case LOAD_ATTR_PROPERTY: + return 1; + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: + return 1; + case _GUARD_DORV_VALUES: + return 1; + case _STORE_ATTR_INSTANCE_VALUE: + return 0; + case STORE_ATTR_INSTANCE_VALUE: + return 0; + case STORE_ATTR_WITH_HINT: + return 0; + case _STORE_ATTR_SLOT: + return 0; + case STORE_ATTR_SLOT: + return 0; + case _SPECIALIZE_COMPARE_OP: + return 2; + case _COMPARE_OP: + return 1; + case COMPARE_OP: + return 1; + case COMPARE_OP_FLOAT: + return 1; + case COMPARE_OP_INT: + return 1; + case COMPARE_OP_STR: + return 1; + case IS_OP: + return 1; + case CONTAINS_OP: + return 1; + case CHECK_EG_MATCH: + return 2; + case CHECK_EXC_MATCH: + return 2; + case IMPORT_NAME: + return 1; + case IMPORT_FROM: + return 2; + case JUMP_FORWARD: + return 0; + case JUMP_BACKWARD: + return 0; + case JUMP: + return 0; + case JUMP_NO_INTERRUPT: + return 0; + case ENTER_EXECUTOR: + return 0; + case _POP_JUMP_IF_FALSE: + return 0; + case _POP_JUMP_IF_TRUE: + return 0; + case _IS_NONE: + return 1; + case POP_JUMP_IF_TRUE: + return 0; + case POP_JUMP_IF_FALSE: + return 0; + case POP_JUMP_IF_NONE: + return 0; + case POP_JUMP_IF_NOT_NONE: + return 0; + case JUMP_BACKWARD_NO_INTERRUPT: + return 0; + case GET_LEN: + return 2; + case MATCH_CLASS: + return 1; + case MATCH_MAPPING: + return 2; + case MATCH_SEQUENCE: + return 2; + case MATCH_KEYS: + return 3; + case GET_ITER: + return 1; + case GET_YIELD_FROM_ITER: + return 1; + case _SPECIALIZE_FOR_ITER: + return 1; + case _FOR_ITER: + return 2; + case _FOR_ITER_TIER_TWO: + return 2; + case FOR_ITER: + return 2; + case INSTRUMENTED_FOR_ITER: + return 0; + case _ITER_CHECK_LIST: + return 1; + case _ITER_JUMP_LIST: + return 1; + case _GUARD_NOT_EXHAUSTED_LIST: + return 1; + case _ITER_NEXT_LIST: + return 2; + case FOR_ITER_LIST: + return 2; + case _ITER_CHECK_TUPLE: + return 1; + case _ITER_JUMP_TUPLE: + return 1; + case _GUARD_NOT_EXHAUSTED_TUPLE: + return 1; + case _ITER_NEXT_TUPLE: + return 2; + case FOR_ITER_TUPLE: + return 2; + case _ITER_CHECK_RANGE: + return 1; + case _ITER_JUMP_RANGE: + return 1; + case _GUARD_NOT_EXHAUSTED_RANGE: + return 1; + case _ITER_NEXT_RANGE: + return 2; + case FOR_ITER_RANGE: + return 2; + case FOR_ITER_GEN: + return 2; + case BEFORE_ASYNC_WITH: + return 2; + case BEFORE_WITH: + return 2; + case WITH_EXCEPT_START: + return 5; + case SETUP_FINALLY: + return 0; + case SETUP_CLEANUP: + return 0; + case SETUP_WITH: + return 0; + case POP_BLOCK: + return 0; + case PUSH_EXC_INFO: + return 2; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_KEYS_VERSION: + return 1; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 2; + case LOAD_ATTR_METHOD_WITH_VALUES: + return 2; + case _LOAD_ATTR_METHOD_NO_DICT: + return 2; + case LOAD_ATTR_METHOD_NO_DICT: + return 2; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 2; + case LOAD_ATTR_METHOD_LAZY_DICT: + return 2; + case INSTRUMENTED_CALL: + return 0; + case _SPECIALIZE_CALL: + return oparg + 2; + case _CALL: + return 1; + case CALL: + return 1; + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return oparg + 2; + case _CHECK_STACK_SPACE: + return oparg + 2; + case _INIT_CALL_PY_EXACT_ARGS: + return 1; + case _PUSH_FRAME: + return 0; + case CALL_BOUND_METHOD_EXACT_ARGS: + return 0; + case CALL_PY_EXACT_ARGS: + return 0; + case CALL_PY_WITH_DEFAULTS: + return 1; + case CALL_TYPE_1: + return 1; + case CALL_STR_1: + return 1; + case CALL_TUPLE_1: + return 1; + case CALL_ALLOC_AND_ENTER_INIT: + return 1; + case EXIT_INIT_CHECK: + return 0; + case CALL_BUILTIN_CLASS: + return 1; + case CALL_BUILTIN_O: + return 1; + case CALL_BUILTIN_FAST: + return 1; + case CALL_BUILTIN_FAST_WITH_KEYWORDS: + return 1; + case CALL_LEN: + return 1; + case CALL_ISINSTANCE: + return 1; + case CALL_LIST_APPEND: + return 1; + case CALL_METHOD_DESCRIPTOR_O: + return 1; + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 1; + case CALL_METHOD_DESCRIPTOR_NOARGS: + return 1; + case CALL_METHOD_DESCRIPTOR_FAST: + return 1; + case INSTRUMENTED_CALL_KW: + return 0; + case CALL_KW: + return 1; + case INSTRUMENTED_CALL_FUNCTION_EX: + return 0; + case CALL_FUNCTION_EX: + return 1; + case MAKE_FUNCTION: + return 1; + case SET_FUNCTION_ATTRIBUTE: + return 1; + case RETURN_GENERATOR: + return 0; + case BUILD_SLICE: + return 1; + case CONVERT_VALUE: + return 1; + case FORMAT_SIMPLE: + return 1; + case FORMAT_WITH_SPEC: + return 1; + case COPY: + return (oparg-1) + 2; + case _SPECIALIZE_BINARY_OP: + return 2; + case _BINARY_OP: + return 1; + case BINARY_OP: + return 1; + case SWAP: + return (oparg-2) + 2; + case INSTRUMENTED_INSTRUCTION: + return 0; + case INSTRUMENTED_JUMP_FORWARD: + return 0; + case INSTRUMENTED_JUMP_BACKWARD: + return 0; + case INSTRUMENTED_POP_JUMP_IF_TRUE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_FALSE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NONE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: + return 0; + case EXTENDED_ARG: + return 0; + case CACHE: + return 0; + case RESERVED: + return 0; + case _GUARD_IS_TRUE_POP: + return 0; + case _GUARD_IS_FALSE_POP: + return 0; + case _GUARD_IS_NONE_POP: + return 0; + case _GUARD_IS_NOT_NONE_POP: + return 0; + case _JUMP_TO_TOP: + return 0; + case _SET_IP: + return 0; + case _SAVE_RETURN_OFFSET: + return 0; + case _EXIT_TRACE: + return 0; + case _INSERT: + return oparg + 1; + case _CHECK_VALIDITY: + return 0; + default: + return -1; + } +} +#endif // NEED_OPCODE_METADATA + +enum InstructionFormat { + INSTR_FMT_IB, + INSTR_FMT_IBC, + INSTR_FMT_IBC0, + INSTR_FMT_IBC00, + INSTR_FMT_IBC000, + INSTR_FMT_IBC0000000, + INSTR_FMT_IBC00000000, + INSTR_FMT_IX, + INSTR_FMT_IXC, + INSTR_FMT_IXC0, + INSTR_FMT_IXC00, + INSTR_FMT_IXC000, +}; + +#define IS_VALID_OPCODE(OP) \ + (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \ + (_PyOpcode_opcode_metadata[(OP)].valid_entry)) + +#define HAS_ARG_FLAG (1) +#define HAS_CONST_FLAG (2) +#define HAS_NAME_FLAG (4) +#define HAS_JUMP_FLAG (8) +#define HAS_FREE_FLAG (16) +#define HAS_LOCAL_FLAG (32) +#define HAS_EVAL_BREAK_FLAG (64) +#define HAS_DEOPT_FLAG (128) +#define HAS_ERROR_FLAG (256) +#define HAS_ESCAPES_FLAG (512) +#define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) +#define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) +#define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) +#define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG)) +#define OPCODE_HAS_FREE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_FREE_FLAG)) +#define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG)) +#define OPCODE_HAS_EVAL_BREAK(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EVAL_BREAK_FLAG)) +#define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG)) +#define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) +#define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) + +struct opcode_metadata { + bool valid_entry; + enum InstructionFormat instr_format; + int flags; +}; + +struct opcode_macro_expansion { + int nuops; + struct { int16_t uop; int8_t size; int8_t offset; } uops[12]; +}; + +#define OPARG_FULL 0 +#define OPARG_CACHE_1 1 +#define OPARG_CACHE_2 2 +#define OPARG_CACHE_4 4 +#define OPARG_TOP 5 +#define OPARG_BOTTOM 6 +#define OPARG_SAVE_RETURN_OFFSET 7 + +#define OPCODE_METADATA_FLAGS(OP) (_PyOpcode_opcode_metadata[(OP)].flags & (HAS_ARG_FLAG | HAS_JUMP_FLAG)) +#define SAME_OPCODE_METADATA(OP1, OP2) \ + (OPCODE_METADATA_FLAGS(OP1) == OPCODE_METADATA_FLAGS(OP2)) + +#define OPCODE_METADATA_SIZE 512 +#define OPCODE_UOP_NAME_SIZE 512 +#define OPCODE_MACRO_EXPANSION_SIZE 256 + +extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]; +#ifdef NEED_OPCODE_METADATA +const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { + [NOP] = { true, INSTR_FMT_IX, 0 }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_CLOSURE] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, 0, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, 0 }, + [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, + [END_FOR] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_SEND] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, + [_SPECIALIZE_TO_BOOL] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_TO_BOOL] = { true, INSTR_FMT_IXC0, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_BOTH_INT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [_BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [_GUARD_BOTH_FLOAT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [_BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [_BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_BINARY_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_STORE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [_POP_FRAME] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_SEND] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [_SEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, + [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_UNPACK_SEQUENCE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_STORE_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, + [_STORE_ATTR] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_LOAD_GLOBAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_GLOBAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [_LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_GLOBAL_BUILTINS] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, + [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [_SPECIALIZE_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ZERO_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ZERO_SUPER_ATTR] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_LOAD_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR] = { true, INSTR_FMT_IBC0000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_TYPE_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_ATTR_MODULE] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_ATTR_WITH_HINT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_ATTR_CLASS] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_GUARD_DORV_VALUES] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_COMPARE_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_NO_INTERRUPT] = { true, 0, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, + [_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_IS_NONE] = { true, INSTR_FMT_IX, 0 }, + [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, + [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, + [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_FOR_ITER_TIER_TWO] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_GUARD_NOT_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_NEXT_LIST] = { true, INSTR_FMT_IX, 0 }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [_ITER_CHECK_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_JUMP_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_GUARD_NOT_EXHAUSTED_TUPLE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_NEXT_TUPLE] = { true, INSTR_FMT_IX, 0 }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [_ITER_CHECK_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_JUMP_RANGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [_GUARD_NOT_EXHAUSTED_RANGE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_ITER_NEXT_RANGE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SETUP_FINALLY] = { true, 0, HAS_ARG_FLAG }, + [SETUP_CLEANUP] = { true, 0, HAS_ARG_FLAG }, + [SETUP_WITH] = { true, 0, HAS_ARG_FLAG }, + [POP_BLOCK] = { true, 0, 0 }, + [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_KEYS_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_SPECIALIZE_CALL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_CALL] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_CHECK_PEP_523] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_CHECK_FUNCTION_EXACT_ARGS] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_CHECK_STACK_SPACE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [_INIT_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_PUSH_FRAME] = { true, INSTR_FMT_IX, 0 }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_SPECIALIZE_BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_BINARY_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [_GUARD_IS_TRUE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_IS_FALSE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_IS_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_GUARD_IS_NOT_NONE_POP] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, + [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, + [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [_SAVE_RETURN_OFFSET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_EXIT_TRACE] = { true, INSTR_FMT_IX, 0 }, + [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_CHECK_VALIDITY] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, +}; +#endif // NEED_OPCODE_METADATA + +extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; +#ifdef NEED_OPCODE_METADATA +const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { + [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, + [RESUME_CHECK] = { .nuops = 1, .uops = { { RESUME_CHECK, 0, 0 } } }, + [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, + [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, + [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, + [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { LOAD_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, + [LOAD_CONST] = { .nuops = 1, .uops = { { LOAD_CONST, 0, 0 } } }, + [STORE_FAST] = { .nuops = 1, .uops = { { STORE_FAST, 0, 0 } } }, + [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { LOAD_FAST, 6, 0 } } }, + [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { STORE_FAST, 5, 0 }, { STORE_FAST, 6, 0 } } }, + [POP_TOP] = { .nuops = 1, .uops = { { POP_TOP, 0, 0 } } }, + [PUSH_NULL] = { .nuops = 1, .uops = { { PUSH_NULL, 0, 0 } } }, + [END_FOR] = { .nuops = 2, .uops = { { POP_TOP, 0, 0 }, { POP_TOP, 0, 0 } } }, + [END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } }, + [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } }, + [UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } }, + [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, + [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } }, + [TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } }, + [TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } }, + [TO_BOOL_NONE] = { .nuops = 1, .uops = { { TO_BOOL_NONE, 0, 0 } } }, + [TO_BOOL_STR] = { .nuops = 1, .uops = { { TO_BOOL_STR, 0, 0 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, + [UNARY_INVERT] = { .nuops = 1, .uops = { { UNARY_INVERT, 0, 0 } } }, + [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, + [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_SUBTRACT_INT, 0, 0 } } }, + [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, 0, 0 } } }, + [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, + [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, + [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, + [BINARY_SUBSCR] = { .nuops = 1, .uops = { { _BINARY_SUBSCR, 0, 0 } } }, + [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } }, + [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, + [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } }, + [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_STR_INT, 0, 0 } } }, + [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, + [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, + [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, + [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } }, + [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, 0, 0 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } }, + [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, + [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, + [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, + [RETURN_CONST] = { .nuops = 2, .uops = { { LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, + [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, + [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, + [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, + [POP_EXCEPT] = { .nuops = 1, .uops = { { POP_EXCEPT, 0, 0 } } }, + [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { LOAD_ASSERTION_ERROR, 0, 0 } } }, + [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } }, + [STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } }, + [DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } }, + [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, 0, 0 } } }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, + [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, + [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } }, + [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } }, + [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, + [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, + [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, + [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, + [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, + [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, + [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, + [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, + [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, + [MAKE_CELL] = { .nuops = 1, .uops = { { MAKE_CELL, 0, 0 } } }, + [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } }, + [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, + [LOAD_DEREF] = { .nuops = 1, .uops = { { LOAD_DEREF, 0, 0 } } }, + [STORE_DEREF] = { .nuops = 1, .uops = { { STORE_DEREF, 0, 0 } } }, + [COPY_FREE_VARS] = { .nuops = 1, .uops = { { COPY_FREE_VARS, 0, 0 } } }, + [BUILD_STRING] = { .nuops = 1, .uops = { { BUILD_STRING, 0, 0 } } }, + [BUILD_TUPLE] = { .nuops = 1, .uops = { { BUILD_TUPLE, 0, 0 } } }, + [BUILD_LIST] = { .nuops = 1, .uops = { { BUILD_LIST, 0, 0 } } }, + [LIST_EXTEND] = { .nuops = 1, .uops = { { LIST_EXTEND, 0, 0 } } }, + [SET_UPDATE] = { .nuops = 1, .uops = { { SET_UPDATE, 0, 0 } } }, + [BUILD_SET] = { .nuops = 1, .uops = { { BUILD_SET, 0, 0 } } }, + [BUILD_MAP] = { .nuops = 1, .uops = { { BUILD_MAP, 0, 0 } } }, + [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { SETUP_ANNOTATIONS, 0, 0 } } }, + [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { BUILD_CONST_KEY_MAP, 0, 0 } } }, + [DICT_UPDATE] = { .nuops = 1, .uops = { { DICT_UPDATE, 0, 0 } } }, + [DICT_MERGE] = { .nuops = 1, .uops = { { DICT_MERGE, 0, 0 } } }, + [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, + [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, + [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, + [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, + [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, + [COMPARE_OP_STR] = { .nuops = 1, .uops = { { COMPARE_OP_STR, 0, 0 } } }, + [IS_OP] = { .nuops = 1, .uops = { { IS_OP, 0, 0 } } }, + [CONTAINS_OP] = { .nuops = 1, .uops = { { CONTAINS_OP, 0, 0 } } }, + [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { CHECK_EG_MATCH, 0, 0 } } }, + [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { CHECK_EXC_MATCH, 0, 0 } } }, + [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, 0, 0 } } }, + [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 0, 0 } } }, + [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 0, 0 } } }, + [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 0, 0 } } }, + [GET_LEN] = { .nuops = 1, .uops = { { GET_LEN, 0, 0 } } }, + [MATCH_CLASS] = { .nuops = 1, .uops = { { MATCH_CLASS, 0, 0 } } }, + [MATCH_MAPPING] = { .nuops = 1, .uops = { { MATCH_MAPPING, 0, 0 } } }, + [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { MATCH_SEQUENCE, 0, 0 } } }, + [MATCH_KEYS] = { .nuops = 1, .uops = { { MATCH_KEYS, 0, 0 } } }, + [GET_ITER] = { .nuops = 1, .uops = { { GET_ITER, 0, 0 } } }, + [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, + [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, 0, 0 } } }, + [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, 0, 0 }, { _ITER_JUMP_LIST, 0, 0 }, { _ITER_NEXT_LIST, 0, 0 } } }, + [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, 0, 0 }, { _ITER_JUMP_TUPLE, 0, 0 }, { _ITER_NEXT_TUPLE, 0, 0 } } }, + [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, 0, 0 }, { _ITER_JUMP_RANGE, 0, 0 }, { _ITER_NEXT_RANGE, 0, 0 } } }, + [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { BEFORE_ASYNC_WITH, 0, 0 } } }, + [BEFORE_WITH] = { .nuops = 1, .uops = { { BEFORE_WITH, 0, 0 } } }, + [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, + [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, + [CALL_STR_1] = { .nuops = 1, .uops = { { CALL_STR_1, 0, 0 } } }, + [CALL_TUPLE_1] = { .nuops = 1, .uops = { { CALL_TUPLE_1, 0, 0 } } }, + [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { EXIT_INIT_CHECK, 0, 0 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { CALL_BUILTIN_CLASS, 0, 0 } } }, + [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { CALL_BUILTIN_O, 0, 0 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST, 0, 0 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, + [CALL_LEN] = { .nuops = 1, .uops = { { CALL_LEN, 0, 0 } } }, + [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { CALL_ISINSTANCE, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, + [MAKE_FUNCTION] = { .nuops = 1, .uops = { { MAKE_FUNCTION, 0, 0 } } }, + [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, + [BUILD_SLICE] = { .nuops = 1, .uops = { { BUILD_SLICE, 0, 0 } } }, + [CONVERT_VALUE] = { .nuops = 1, .uops = { { CONVERT_VALUE, 0, 0 } } }, + [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } }, + [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } }, + [COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } }, + [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, + [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, +}; +#endif // NEED_OPCODE_METADATA + +extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; +#ifdef NEED_OPCODE_METADATA +const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { + [_EXIT_TRACE] = "_EXIT_TRACE", + [_SET_IP] = "_SET_IP", + [_SPECIALIZE_TO_BOOL] = "_SPECIALIZE_TO_BOOL", + [_TO_BOOL] = "_TO_BOOL", + [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", + [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", + [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", + [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", + [_SPECIALIZE_BINARY_SUBSCR] = "_SPECIALIZE_BINARY_SUBSCR", + [_BINARY_SUBSCR] = "_BINARY_SUBSCR", + [_SPECIALIZE_STORE_SUBSCR] = "_SPECIALIZE_STORE_SUBSCR", + [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_POP_FRAME] = "_POP_FRAME", + [_SPECIALIZE_SEND] = "_SPECIALIZE_SEND", + [_SEND] = "_SEND", + [_SPECIALIZE_UNPACK_SEQUENCE] = "_SPECIALIZE_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_SPECIALIZE_STORE_ATTR] = "_SPECIALIZE_STORE_ATTR", + [_STORE_ATTR] = "_STORE_ATTR", + [_SPECIALIZE_LOAD_GLOBAL] = "_SPECIALIZE_LOAD_GLOBAL", + [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", + [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_SPECIALIZE_LOAD_SUPER_ATTR] = "_SPECIALIZE_LOAD_SUPER_ATTR", + [_LOAD_SUPER_ATTR] = "_LOAD_SUPER_ATTR", + [_SPECIALIZE_LOAD_ATTR] = "_SPECIALIZE_LOAD_ATTR", + [_LOAD_ATTR] = "_LOAD_ATTR", + [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", + [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", + [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", + [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_SPECIALIZE_COMPARE_OP] = "_SPECIALIZE_COMPARE_OP", + [_COMPARE_OP] = "_COMPARE_OP", + [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", + [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", + [_IS_NONE] = "_IS_NONE", + [_SPECIALIZE_FOR_ITER] = "_SPECIALIZE_FOR_ITER", + [_FOR_ITER] = "_FOR_ITER", + [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", + [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST", + [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", + [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", + [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_JUMP_TUPLE] = "_ITER_JUMP_TUPLE", + [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_JUMP_RANGE] = "_ITER_JUMP_RANGE", + [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_SPECIALIZE_CALL] = "_SPECIALIZE_CALL", + [_CALL] = "_CALL", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_PUSH_FRAME] = "_PUSH_FRAME", + [_SPECIALIZE_BINARY_OP] = "_SPECIALIZE_BINARY_OP", + [_BINARY_OP] = "_BINARY_OP", + [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", + [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_INSERT] = "_INSERT", + [_CHECK_VALIDITY] = "_CHECK_VALIDITY", +}; +#endif // NEED_OPCODE_METADATA + +extern const char *const _PyOpcode_OpName[268]; +#ifdef NEED_OPCODE_METADATA +const char *const _PyOpcode_OpName[268] = { + [CACHE] = "CACHE", + [RESERVED] = "RESERVED", + [RESUME] = "RESUME", + [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", + [BEFORE_WITH] = "BEFORE_WITH", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", + [BINARY_SLICE] = "BINARY_SLICE", + [BINARY_SUBSCR] = "BINARY_SUBSCR", + [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", + [CLEANUP_THROW] = "CLEANUP_THROW", + [DELETE_SUBSCR] = "DELETE_SUBSCR", + [END_ASYNC_FOR] = "END_ASYNC_FOR", + [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", + [FORMAT_SIMPLE] = "FORMAT_SIMPLE", + [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", + [GET_AITER] = "GET_AITER", + [GET_ANEXT] = "GET_ANEXT", + [GET_ITER] = "GET_ITER", + [GET_LEN] = "GET_LEN", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", + [INTERPRETER_EXIT] = "INTERPRETER_EXIT", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_LOCALS] = "LOAD_LOCALS", + [MAKE_FUNCTION] = "MAKE_FUNCTION", + [MATCH_KEYS] = "MATCH_KEYS", + [MATCH_MAPPING] = "MATCH_MAPPING", + [MATCH_SEQUENCE] = "MATCH_SEQUENCE", + [NOP] = "NOP", + [POP_EXCEPT] = "POP_EXCEPT", + [POP_TOP] = "POP_TOP", + [PUSH_EXC_INFO] = "PUSH_EXC_INFO", + [PUSH_NULL] = "PUSH_NULL", + [RETURN_GENERATOR] = "RETURN_GENERATOR", + [RETURN_VALUE] = "RETURN_VALUE", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [STORE_SLICE] = "STORE_SLICE", + [STORE_SUBSCR] = "STORE_SUBSCR", + [TO_BOOL] = "TO_BOOL", + [UNARY_INVERT] = "UNARY_INVERT", + [UNARY_NEGATIVE] = "UNARY_NEGATIVE", + [UNARY_NOT] = "UNARY_NOT", + [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [BINARY_OP] = "BINARY_OP", + [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", + [BUILD_LIST] = "BUILD_LIST", + [BUILD_MAP] = "BUILD_MAP", + [BUILD_SET] = "BUILD_SET", + [BUILD_SLICE] = "BUILD_SLICE", + [BUILD_STRING] = "BUILD_STRING", + [BUILD_TUPLE] = "BUILD_TUPLE", + [CALL] = "CALL", + [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", + [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", + [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", + [CALL_KW] = "CALL_KW", + [COMPARE_OP] = "COMPARE_OP", + [CONTAINS_OP] = "CONTAINS_OP", + [CONVERT_VALUE] = "CONVERT_VALUE", + [COPY] = "COPY", + [COPY_FREE_VARS] = "COPY_FREE_VARS", + [DELETE_ATTR] = "DELETE_ATTR", + [DELETE_DEREF] = "DELETE_DEREF", + [DELETE_FAST] = "DELETE_FAST", + [DELETE_GLOBAL] = "DELETE_GLOBAL", + [DELETE_NAME] = "DELETE_NAME", + [DICT_MERGE] = "DICT_MERGE", + [DICT_UPDATE] = "DICT_UPDATE", + [ENTER_EXECUTOR] = "ENTER_EXECUTOR", + [EXTENDED_ARG] = "EXTENDED_ARG", + [FOR_ITER] = "FOR_ITER", + [GET_AWAITABLE] = "GET_AWAITABLE", + [IMPORT_FROM] = "IMPORT_FROM", + [IMPORT_NAME] = "IMPORT_NAME", + [IS_OP] = "IS_OP", + [JUMP_BACKWARD] = "JUMP_BACKWARD", + [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", + [JUMP_FORWARD] = "JUMP_FORWARD", + [LIST_APPEND] = "LIST_APPEND", + [LIST_EXTEND] = "LIST_EXTEND", + [LOAD_ATTR] = "LOAD_ATTR", + [LOAD_CONST] = "LOAD_CONST", + [LOAD_DEREF] = "LOAD_DEREF", + [LOAD_FAST] = "LOAD_FAST", + [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", + [LOAD_FAST_CHECK] = "LOAD_FAST_CHECK", + [LOAD_FAST_LOAD_FAST] = "LOAD_FAST_LOAD_FAST", + [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", + [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", + [LOAD_GLOBAL] = "LOAD_GLOBAL", + [LOAD_NAME] = "LOAD_NAME", + [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", + [MAKE_CELL] = "MAKE_CELL", + [MAP_ADD] = "MAP_ADD", + [MATCH_CLASS] = "MATCH_CLASS", + [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", + [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", + [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", + [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", + [RAISE_VARARGS] = "RAISE_VARARGS", + [RERAISE] = "RERAISE", + [RETURN_CONST] = "RETURN_CONST", + [SEND] = "SEND", + [SET_ADD] = "SET_ADD", + [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", + [SET_UPDATE] = "SET_UPDATE", + [STORE_ATTR] = "STORE_ATTR", + [STORE_DEREF] = "STORE_DEREF", + [STORE_FAST] = "STORE_FAST", + [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", + [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", + [STORE_GLOBAL] = "STORE_GLOBAL", + [STORE_NAME] = "STORE_NAME", + [SWAP] = "SWAP", + [UNPACK_EX] = "UNPACK_EX", + [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", + [YIELD_VALUE] = "YIELD_VALUE", + [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", + [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", + [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", + [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", + [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", + [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", + [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", + [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_O] = "CALL_BUILTIN_O", + [CALL_ISINSTANCE] = "CALL_ISINSTANCE", + [CALL_LEN] = "CALL_LEN", + [CALL_LIST_APPEND] = "CALL_LIST_APPEND", + [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", + [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_STR_1] = "CALL_STR_1", + [CALL_TUPLE_1] = "CALL_TUPLE_1", + [CALL_TYPE_1] = "CALL_TYPE_1", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [COMPARE_OP_INT] = "COMPARE_OP_INT", + [COMPARE_OP_STR] = "COMPARE_OP_STR", + [FOR_ITER_GEN] = "FOR_ITER_GEN", + [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", + [RESUME_CHECK] = "RESUME_CHECK", + [SEND_GEN] = "SEND_GEN", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", + [TO_BOOL_BOOL] = "TO_BOOL_BOOL", + [TO_BOOL_INT] = "TO_BOOL_INT", + [TO_BOOL_LIST] = "TO_BOOL_LIST", + [TO_BOOL_NONE] = "TO_BOOL_NONE", + [TO_BOOL_STR] = "TO_BOOL_STR", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_CALL_KW] = "INSTRUMENTED_CALL_KW", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", + [JUMP] = "JUMP", + [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", + [LOAD_CLOSURE] = "LOAD_CLOSURE", + [LOAD_METHOD] = "LOAD_METHOD", + [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", + [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", + [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", + [POP_BLOCK] = "POP_BLOCK", + [SETUP_CLEANUP] = "SETUP_CLEANUP", + [SETUP_FINALLY] = "SETUP_FINALLY", + [SETUP_WITH] = "SETUP_WITH", + [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", +}; +#endif // NEED_OPCODE_METADATA + +extern const uint8_t _PyOpcode_Caches[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_Caches[256] = { + [TO_BOOL] = 3, + [BINARY_OP_INPLACE_ADD_UNICODE] = 1, + [BINARY_SUBSCR] = 1, + [STORE_SUBSCR] = 1, + [SEND] = 1, + [UNPACK_SEQUENCE] = 1, + [STORE_ATTR] = 4, + [LOAD_GLOBAL] = 4, + [LOAD_SUPER_ATTR] = 1, + [LOAD_ATTR] = 9, + [COMPARE_OP] = 1, + [JUMP_BACKWARD] = 1, + [POP_JUMP_IF_TRUE] = 1, + [POP_JUMP_IF_FALSE] = 1, + [POP_JUMP_IF_NONE] = 1, + [POP_JUMP_IF_NOT_NONE] = 1, + [FOR_ITER] = 1, + [CALL] = 3, + [BINARY_OP] = 1, +}; +#endif // NEED_OPCODE_METADATA + +extern const uint8_t _PyOpcode_Deopt[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_Deopt[256] = { + [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, + [BEFORE_WITH] = BEFORE_WITH, + [BINARY_OP] = BINARY_OP, + [BINARY_OP_ADD_FLOAT] = BINARY_OP, + [BINARY_OP_ADD_INT] = BINARY_OP, + [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, + [BINARY_OP_MULTIPLY_INT] = BINARY_OP, + [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, + [BINARY_OP_SUBTRACT_INT] = BINARY_OP, + [BINARY_SLICE] = BINARY_SLICE, + [BINARY_SUBSCR] = BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, + [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, + [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, + [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, + [BUILD_LIST] = BUILD_LIST, + [BUILD_MAP] = BUILD_MAP, + [BUILD_SET] = BUILD_SET, + [BUILD_SLICE] = BUILD_SLICE, + [BUILD_STRING] = BUILD_STRING, + [BUILD_TUPLE] = BUILD_TUPLE, + [CACHE] = CACHE, + [CALL] = CALL, + [CALL_ALLOC_AND_ENTER_INIT] = CALL, + [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BUILTIN_CLASS] = CALL, + [CALL_BUILTIN_FAST] = CALL, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, + [CALL_BUILTIN_O] = CALL, + [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = CALL, + [CALL_KW] = CALL_KW, + [CALL_LEN] = CALL, + [CALL_LIST_APPEND] = CALL, + [CALL_METHOD_DESCRIPTOR_FAST] = CALL, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, + [CALL_METHOD_DESCRIPTOR_NOARGS] = CALL, + [CALL_METHOD_DESCRIPTOR_O] = CALL, + [CALL_PY_EXACT_ARGS] = CALL, + [CALL_PY_WITH_DEFAULTS] = CALL, + [CALL_STR_1] = CALL, + [CALL_TUPLE_1] = CALL, + [CALL_TYPE_1] = CALL, + [CHECK_EG_MATCH] = CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, + [CLEANUP_THROW] = CLEANUP_THROW, + [COMPARE_OP] = COMPARE_OP, + [COMPARE_OP_FLOAT] = COMPARE_OP, + [COMPARE_OP_INT] = COMPARE_OP, + [COMPARE_OP_STR] = COMPARE_OP, + [CONTAINS_OP] = CONTAINS_OP, + [CONVERT_VALUE] = CONVERT_VALUE, + [COPY] = COPY, + [COPY_FREE_VARS] = COPY_FREE_VARS, + [DELETE_ATTR] = DELETE_ATTR, + [DELETE_DEREF] = DELETE_DEREF, + [DELETE_FAST] = DELETE_FAST, + [DELETE_GLOBAL] = DELETE_GLOBAL, + [DELETE_NAME] = DELETE_NAME, + [DELETE_SUBSCR] = DELETE_SUBSCR, + [DICT_MERGE] = DICT_MERGE, + [DICT_UPDATE] = DICT_UPDATE, + [END_ASYNC_FOR] = END_ASYNC_FOR, + [END_FOR] = END_FOR, + [END_SEND] = END_SEND, + [ENTER_EXECUTOR] = ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, + [EXTENDED_ARG] = EXTENDED_ARG, + [FORMAT_SIMPLE] = FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = FORMAT_WITH_SPEC, + [FOR_ITER] = FOR_ITER, + [FOR_ITER_GEN] = FOR_ITER, + [FOR_ITER_LIST] = FOR_ITER, + [FOR_ITER_RANGE] = FOR_ITER, + [FOR_ITER_TUPLE] = FOR_ITER, + [GET_AITER] = GET_AITER, + [GET_ANEXT] = GET_ANEXT, + [GET_AWAITABLE] = GET_AWAITABLE, + [GET_ITER] = GET_ITER, + [GET_LEN] = GET_LEN, + [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, + [IMPORT_FROM] = IMPORT_FROM, + [IMPORT_NAME] = IMPORT_NAME, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = INTERPRETER_EXIT, + [IS_OP] = IS_OP, + [JUMP_BACKWARD] = JUMP_BACKWARD, + [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_FORWARD] = JUMP_FORWARD, + [LIST_APPEND] = LIST_APPEND, + [LIST_EXTEND] = LIST_EXTEND, + [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, + [LOAD_ATTR] = LOAD_ATTR, + [LOAD_ATTR_CLASS] = LOAD_ATTR, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, + [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, + [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, + [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, + [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, + [LOAD_ATTR_MODULE] = LOAD_ATTR, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = LOAD_ATTR, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = LOAD_ATTR, + [LOAD_ATTR_PROPERTY] = LOAD_ATTR, + [LOAD_ATTR_SLOT] = LOAD_ATTR, + [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, + [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_CONST] = LOAD_CONST, + [LOAD_DEREF] = LOAD_DEREF, + [LOAD_FAST] = LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, + [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, + [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, + [LOAD_LOCALS] = LOAD_LOCALS, + [LOAD_NAME] = LOAD_NAME, + [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, + [MAKE_CELL] = MAKE_CELL, + [MAKE_FUNCTION] = MAKE_FUNCTION, + [MAP_ADD] = MAP_ADD, + [MATCH_CLASS] = MATCH_CLASS, + [MATCH_KEYS] = MATCH_KEYS, + [MATCH_MAPPING] = MATCH_MAPPING, + [MATCH_SEQUENCE] = MATCH_SEQUENCE, + [NOP] = NOP, + [POP_EXCEPT] = POP_EXCEPT, + [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, + [POP_TOP] = POP_TOP, + [PUSH_EXC_INFO] = PUSH_EXC_INFO, + [PUSH_NULL] = PUSH_NULL, + [RAISE_VARARGS] = RAISE_VARARGS, + [RERAISE] = RERAISE, + [RESERVED] = RESERVED, + [RESUME] = RESUME, + [RESUME_CHECK] = RESUME, + [RETURN_CONST] = RETURN_CONST, + [RETURN_GENERATOR] = RETURN_GENERATOR, + [RETURN_VALUE] = RETURN_VALUE, + [SEND] = SEND, + [SEND_GEN] = SEND, + [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, + [SET_ADD] = SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = SET_UPDATE, + [STORE_ATTR] = STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, + [STORE_ATTR_SLOT] = STORE_ATTR, + [STORE_ATTR_WITH_HINT] = STORE_ATTR, + [STORE_DEREF] = STORE_DEREF, + [STORE_FAST] = STORE_FAST, + [STORE_FAST_LOAD_FAST] = STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = STORE_GLOBAL, + [STORE_NAME] = STORE_NAME, + [STORE_SLICE] = STORE_SLICE, + [STORE_SUBSCR] = STORE_SUBSCR, + [STORE_SUBSCR_DICT] = STORE_SUBSCR, + [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, + [SWAP] = SWAP, + [TO_BOOL] = TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, + [TO_BOOL_BOOL] = TO_BOOL, + [TO_BOOL_INT] = TO_BOOL, + [TO_BOOL_LIST] = TO_BOOL, + [TO_BOOL_NONE] = TO_BOOL, + [TO_BOOL_STR] = TO_BOOL, + [UNARY_INVERT] = UNARY_INVERT, + [UNARY_NEGATIVE] = UNARY_NEGATIVE, + [UNARY_NOT] = UNARY_NOT, + [UNPACK_EX] = UNPACK_EX, + [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, + [WITH_EXCEPT_START] = WITH_EXCEPT_START, + [YIELD_VALUE] = YIELD_VALUE, +}; +#endif // NEED_OPCODE_METADATA + +#define EXTRA_CASES \ + case 119: \ + case 120: \ + case 121: \ + case 122: \ + case 123: \ + case 124: \ + case 125: \ + case 126: \ + case 127: \ + case 128: \ + case 129: \ + case 130: \ + case 131: \ + case 132: \ + case 133: \ + case 134: \ + case 135: \ + case 136: \ + case 137: \ + case 138: \ + case 139: \ + case 140: \ + case 141: \ + case 142: \ + case 143: \ + case 144: \ + case 145: \ + case 146: \ + case 147: \ + case 148: \ + case 219: \ + case 220: \ + case 221: \ + case 222: \ + case 223: \ + case 224: \ + case 225: \ + case 226: \ + case 227: \ + case 228: \ + case 229: \ + case 230: \ + case 231: \ + case 232: \ + case 233: \ + case 234: \ + case 235: \ + case 255: \ + ; + diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 1d5ff988290bd4..208bfb2f75308b 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -8,17 +8,13 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode.h" // _PyOpcode_Jump - +#include "opcode_ids.h" #define MAX_REAL_OPCODE 254 #define IS_WITHIN_OPCODE_RANGE(opcode) \ (((opcode) >= 0 && (opcode) <= MAX_REAL_OPCODE) || \ - IS_PSEUDO_OPCODE(opcode)) - -#define IS_JUMP_OPCODE(opcode) \ - is_bit_set_in_table(_PyOpcode_Jump, opcode) + IS_PSEUDO_INSTR(opcode)) #define IS_BLOCK_PUSH_OPCODE(opcode) \ ((opcode) == SETUP_FINALLY || \ @@ -26,11 +22,11 @@ extern "C" { (opcode) == SETUP_CLEANUP) #define HAS_TARGET(opcode) \ - (IS_JUMP_OPCODE(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)) + (OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)) /* opcodes that must be last in the basicblock */ #define IS_TERMINATOR_OPCODE(opcode) \ - (IS_JUMP_OPCODE(opcode) || IS_SCOPE_EXIT_OPCODE(opcode)) + (OPCODE_HAS_JUMP(opcode) || IS_SCOPE_EXIT_OPCODE(opcode)) /* opcodes which are not emitted in codegen stage, only by the assembler */ #define IS_ASSEMBLER_OPCODE(opcode) \ @@ -55,36 +51,21 @@ extern "C" { (opcode) == RAISE_VARARGS || \ (opcode) == RERAISE) -#define IS_SUPERINSTRUCTION_OPCODE(opcode) \ - ((opcode) == LOAD_FAST__LOAD_FAST || \ - (opcode) == LOAD_FAST__LOAD_CONST || \ - (opcode) == LOAD_CONST__LOAD_FAST || \ - (opcode) == STORE_FAST__LOAD_FAST || \ - (opcode) == STORE_FAST__STORE_FAST) - - -#define LOG_BITS_PER_INT 5 -#define MASK_LOW_LOG_BITS 31 - -static inline int -is_bit_set_in_table(const uint32_t *table, int bitindex) { - /* Is the relevant bit set in the relevant word? */ - /* 512 bits fit into 9 32-bits words. - * Word is indexed by (bitindex>>ln(size of int in bits)). - * Bit within word is the low bits of bitindex. - */ - if (bitindex >= 0 && bitindex < 512) { - uint32_t word = table[bitindex >> LOG_BITS_PER_INT]; - return (word >> (bitindex & MASK_LOW_LOG_BITS)) & 1; - } - else { - return 0; - } -} -#undef LOG_BITS_PER_INT -#undef MASK_LOW_LOG_BITS +/* Flags used in the oparg for MAKE_FUNCTION */ +#define MAKE_FUNCTION_DEFAULTS 0x01 +#define MAKE_FUNCTION_KWDEFAULTS 0x02 +#define MAKE_FUNCTION_ANNOTATIONS 0x04 +#define MAKE_FUNCTION_CLOSURE 0x08 + +/* Values used in the oparg for RESUME */ +#define RESUME_AT_FUNC_START 0 +#define RESUME_AFTER_YIELD 1 +#define RESUME_AFTER_YIELD_FROM 2 +#define RESUME_AFTER_AWAIT 3 +#define RESUME_OPARG_LOCATION_MASK 0x3 +#define RESUME_OPARG_DEPTH1_MASK 0x4 #ifdef __cplusplus } diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h new file mode 100644 index 00000000000000..b052460b44b791 --- /dev/null +++ b/Include/internal/pycore_optimizer.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_OPTIMIZER_H +#define Py_INTERNAL_OPTIMIZER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_uops.h" // _PyUOpInstruction + +int _Py_uop_analyze_and_optimize(PyCodeObject *code, + _PyUOpInstruction *trace, int trace_len, int curr_stackentries); + +extern PyTypeObject _PyCounterExecutor_Type; +extern PyTypeObject _PyCounterOptimizer_Type; +extern PyTypeObject _PyDefaultOptimizer_Type; +extern PyTypeObject _PyUOpExecutor_Type; +extern PyTypeObject _PyUOpOptimizer_Type; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OPTIMIZER_H */ diff --git a/Include/internal/pycore_parking_lot.h b/Include/internal/pycore_parking_lot.h new file mode 100644 index 00000000000000..f444da730055e8 --- /dev/null +++ b/Include/internal/pycore_parking_lot.h @@ -0,0 +1,99 @@ +// ParkingLot is an internal API for building efficient synchronization +// primitives like mutexes and events. +// +// The API and name is inspired by WebKit's WTF::ParkingLot, which in turn +// is inspired Linux's futex API. +// See https://webkit.org/blog/6161/locking-in-webkit/. +// +// The core functionality is an atomic "compare-and-sleep" operation along with +// an atomic "wake-up" operation. + +#ifndef Py_INTERNAL_PARKING_LOT_H +#define Py_INTERNAL_PARKING_LOT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_time.h" // _PyTime_t + + +enum { + // The thread was unparked by another thread. + Py_PARK_OK = 0, + + // The value of `address` did not match `expected`. + Py_PARK_AGAIN = -1, + + // The thread was unparked due to a timeout. + Py_PARK_TIMEOUT = -2, + + // The thread was interrupted by a signal. + Py_PARK_INTR = -3, +}; + +// Checks that `*address == *expected` and puts the thread to sleep until an +// unpark operation is called on the same `address`. Otherwise, the function +// returns `Py_PARK_AGAIN`. The comparison behaves like memcmp, but is +// performed atomically with respect to unpark operations. +// +// The `address_size` argument is the size of the data pointed to by the +// `address` and `expected` pointers (i.e., sizeof(*address)). It must be +// 1, 2, 4, or 8. +// +// The `timeout_ns` argument specifies the maximum amount of time to wait, with +// -1 indicating an infinite wait. +// +// `park_arg`, which can be NULL, is passed to the unpark operation. +// +// If `detach` is true, then the thread will detach/release the GIL while +// waiting. +// +// Example usage: +// +// if (_Py_atomic_compare_exchange_uint8(address, &expected, new_value)) { +// int res = _PyParkingLot_Park(address, &new_value, sizeof(*address), +// timeout_ns, NULL, 1); +// ... +// } +PyAPI_FUNC(int) +_PyParkingLot_Park(const void *address, const void *expected, + size_t address_size, _PyTime_t timeout_ns, + void *park_arg, int detach); + +// Callback for _PyParkingLot_Unpark: +// +// `arg` is the data of the same name provided to the _PyParkingLot_Unpark() +// call. +// `park_arg` is the data provided to _PyParkingLot_Park() call or NULL if +// no waiting thread was found. +// `has_more_waiters` is true if there are more threads waiting on the same +// address. May be true in cases where threads are waiting on a different +// address that map to the same internal bucket. +typedef void _Py_unpark_fn_t(void *arg, void *park_arg, int has_more_waiters); + +// Unparks a single thread waiting on `address`. +// +// Note that fn() is called regardless of whether a thread was unparked. If +// no threads are waiting on `address` then the `park_arg` argument to fn() +// will be NULL. +// +// Example usage: +// void callback(void *arg, void *park_arg, int has_more_waiters); +// _PyParkingLot_Unpark(address, &callback, arg); +PyAPI_FUNC(void) +_PyParkingLot_Unpark(const void *address, _Py_unpark_fn_t *fn, void *arg); + +// Unparks all threads waiting on `address`. +PyAPI_FUNC(void) _PyParkingLot_UnparkAll(const void *address); + +// Resets the parking lot state after a fork. Forgets all parked threads. +PyAPI_FUNC(void) _PyParkingLot_AfterFork(void); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PARKING_LOT_H */ diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index dd51b92801aebf..067b34c12c4e7f 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -58,7 +58,17 @@ extern struct _mod* _PyParser_ASTFromFile( PyCompilerFlags *flags, int *errcode, PyArena *arena); - +extern struct _mod* _PyParser_InteractiveASTFromFile( + FILE *fp, + PyObject *filename_ob, + const char *enc, + int mode, + const char *ps1, + const char *ps2, + PyCompilerFlags *flags, + int *errcode, + PyObject **interactive_src, + PyArena *arena); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index b8deaa0c3eb067..a1ce1b19a00283 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -8,7 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); + extern PyStatus _PyPathConfig_ReadGlobal(PyConfig *config); extern PyStatus _PyPathConfig_UpdateGlobal(const PyConfig *config); extern const wchar_t * _PyPathConfig_GetGlobalModuleSearchPath(void); diff --git a/Include/internal/pycore_pyarena.h b/Include/internal/pycore_pyarena.h index d78972a88ca238..1f07479fb2ca27 100644 --- a/Include/internal/pycore_pyarena.h +++ b/Include/internal/pycore_pyarena.h @@ -1,5 +1,4 @@ -/* An arena-like memory interface for the compiler. - */ +// An arena-like memory interface for the compiler. #ifndef Py_INTERNAL_PYARENA_H #define Py_INTERNAL_PYARENA_H @@ -13,49 +12,54 @@ extern "C" { typedef struct _arena PyArena; -/* _PyArena_New() and _PyArena_Free() create a new arena and free it, - respectively. Once an arena has been created, it can be used - to allocate memory via _PyArena_Malloc(). Pointers to PyObject can - also be registered with the arena via _PyArena_AddPyObject(), and the - arena will ensure that the PyObjects stay alive at least until - _PyArena_Free() is called. When an arena is freed, all the memory it - allocated is freed, the arena releases internal references to registered - PyObject*, and none of its pointers are valid. - XXX (tim) What does "none of its pointers are valid" mean? Does it - XXX mean that pointers previously obtained via _PyArena_Malloc() are - XXX no longer valid? (That's clearly true, but not sure that's what - XXX the text is trying to say.) - - _PyArena_New() returns an arena pointer. On error, it - returns a negative number and sets an exception. - XXX (tim): Not true. On error, _PyArena_New() actually returns NULL, - XXX and looks like it may or may not set an exception (e.g., if the - XXX internal PyList_New(0) returns NULL, _PyArena_New() passes that on - XXX and an exception is set; OTOH, if the internal - XXX block_new(DEFAULT_BLOCK_SIZE) returns NULL, that's passed on but - XXX an exception is not set in that case). -*/ +// _PyArena_New() and _PyArena_Free() create a new arena and free it, +// respectively. Once an arena has been created, it can be used +// to allocate memory via _PyArena_Malloc(). Pointers to PyObject can +// also be registered with the arena via _PyArena_AddPyObject(), and the +// arena will ensure that the PyObjects stay alive at least until +// _PyArena_Free() is called. When an arena is freed, all the memory it +// allocated is freed, the arena releases internal references to registered +// PyObject*, and none of its pointers are valid. +// XXX (tim) What does "none of its pointers are valid" mean? Does it +// XXX mean that pointers previously obtained via _PyArena_Malloc() are +// XXX no longer valid? (That's clearly true, but not sure that's what +// XXX the text is trying to say.) +// +// _PyArena_New() returns an arena pointer. On error, it +// returns a negative number and sets an exception. +// XXX (tim): Not true. On error, _PyArena_New() actually returns NULL, +// XXX and looks like it may or may not set an exception (e.g., if the +// XXX internal PyList_New(0) returns NULL, _PyArena_New() passes that on +// XXX and an exception is set; OTOH, if the internal +// XXX block_new(DEFAULT_BLOCK_SIZE) returns NULL, that's passed on but +// XXX an exception is not set in that case). +// +// Export for test_peg_generator PyAPI_FUNC(PyArena*) _PyArena_New(void); + +// Export for test_peg_generator PyAPI_FUNC(void) _PyArena_Free(PyArena *); -/* Mostly like malloc(), return the address of a block of memory spanning - * `size` bytes, or return NULL (without setting an exception) if enough - * new memory can't be obtained. Unlike malloc(0), _PyArena_Malloc() with - * size=0 does not guarantee to return a unique pointer (the pointer - * returned may equal one or more other pointers obtained from - * _PyArena_Malloc()). - * Note that pointers obtained via _PyArena_Malloc() must never be passed to - * the system free() or realloc(), or to any of Python's similar memory- - * management functions. _PyArena_Malloc()-obtained pointers remain valid - * until _PyArena_Free(ar) is called, at which point all pointers obtained - * from the arena `ar` become invalid simultaneously. - */ +// Mostly like malloc(), return the address of a block of memory spanning +// `size` bytes, or return NULL (without setting an exception) if enough +// new memory can't be obtained. Unlike malloc(0), _PyArena_Malloc() with +// size=0 does not guarantee to return a unique pointer (the pointer +// returned may equal one or more other pointers obtained from +// _PyArena_Malloc()). +// Note that pointers obtained via _PyArena_Malloc() must never be passed to +// the system free() or realloc(), or to any of Python's similar memory- +// management functions. _PyArena_Malloc()-obtained pointers remain valid +// until _PyArena_Free(ar) is called, at which point all pointers obtained +// from the arena `ar` become invalid simultaneously. +// +// Export for test_peg_generator PyAPI_FUNC(void*) _PyArena_Malloc(PyArena *, size_t size); -/* This routine isn't a proper arena allocation routine. It takes - * a PyObject* and records it so that it can be DECREFed when the - * arena is freed. - */ +// This routine isn't a proper arena allocation routine. It takes +// a PyObject* and records it so that it can be DECREFed when the +// arena is freed. +// +// Export for test_peg_generator PyAPI_FUNC(int) _PyArena_AddPyObject(PyArena *, PyObject *); #ifdef __cplusplus diff --git a/Include/internal/pycore_pybuffer.h b/Include/internal/pycore_pybuffer.h new file mode 100644 index 00000000000000..3cbc290b2ea3ee --- /dev/null +++ b/Include/internal/pycore_pybuffer.h @@ -0,0 +1,21 @@ +#ifndef Py_INTERNAL_PYBUFFER_H +#define Py_INTERNAL_PYBUFFER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +// Exported for the _xxinterpchannels module. +PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreter( + PyInterpreterState *interp, Py_buffer *view); +PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreterAndRawFree( + PyInterpreterState *interp, Py_buffer *view); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYBUFFER_H */ diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 4620a269644917..0f16fb894d17e1 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -9,6 +9,59 @@ extern "C" { #endif +/* Error handling definitions */ + +extern _PyErr_StackItem* _PyErr_GetTopmostException(PyThreadState *tstate); +extern PyObject* _PyErr_GetHandledException(PyThreadState *); +extern void _PyErr_SetHandledException(PyThreadState *, PyObject *); +extern void _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); + +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *); + + +// Like PyErr_Format(), but saves current exception as __context__ and +// __cause__. +// Export for '_sqlite3' shared extension. +PyAPI_FUNC(PyObject*) _PyErr_FormatFromCause( + PyObject *exception, + const char *format, /* ASCII-encoded string */ + ... + ); + +extern int _PyException_AddNote( + PyObject *exc, + PyObject *note); + +extern int _PyErr_CheckSignals(void); + +/* Support for adding program text to SyntaxErrors */ + +// Export for test_peg_generator +PyAPI_FUNC(PyObject*) _PyErr_ProgramDecodedTextObject( + PyObject *filename, + int lineno, + const char* encoding); + +extern PyObject* _PyUnicodeTranslateError_Create( + PyObject *object, + Py_ssize_t start, + Py_ssize_t end, + const char *reason /* UTF-8 encoded string */ + ); + +extern void _Py_NO_RETURN _Py_FatalErrorFormat( + const char *func, + const char *format, + ...); + +extern PyObject* _PyErr_SetImportErrorWithNameFrom( + PyObject *, + PyObject *, + PyObject *, + PyObject *); + + /* runtime lifecycle */ extern PyStatus _PyErr_InitTypes(PyInterpreterState *); @@ -31,86 +84,92 @@ static inline void _PyErr_ClearExcState(_PyErr_StackItem *exc_state) Py_CLEAR(exc_state->exc_value); } -PyAPI_FUNC(PyObject*) _PyErr_StackItemToExcInfoTuple( +extern PyObject* _PyErr_StackItemToExcInfoTuple( _PyErr_StackItem *err_info); -PyAPI_FUNC(void) _PyErr_Fetch( +extern void _PyErr_Fetch( PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **traceback); -extern PyObject * -_PyErr_GetRaisedException(PyThreadState *tstate); +extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate); -PyAPI_FUNC(int) _PyErr_ExceptionMatches( +extern int _PyErr_ExceptionMatches( PyThreadState *tstate, PyObject *exc); -void -_PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc); +extern void _PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc); -PyAPI_FUNC(void) _PyErr_Restore( +extern void _PyErr_Restore( PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *traceback); -PyAPI_FUNC(void) _PyErr_SetObject( +extern void _PyErr_SetObject( PyThreadState *tstate, PyObject *type, PyObject *value); -PyAPI_FUNC(void) _PyErr_ChainStackItem( - _PyErr_StackItem *exc_info); +extern void _PyErr_ChainStackItem(void); -PyAPI_FUNC(void) _PyErr_Clear(PyThreadState *tstate); +extern void _PyErr_Clear(PyThreadState *tstate); -PyAPI_FUNC(void) _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); +extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); -PyAPI_FUNC(PyObject *) _PyErr_NoMemory(PyThreadState *tstate); +extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); -PyAPI_FUNC(void) _PyErr_SetString( +extern void _PyErr_SetString( PyThreadState *tstate, PyObject *exception, const char *string); -PyAPI_FUNC(PyObject *) _PyErr_Format( +extern PyObject* _PyErr_Format( PyThreadState *tstate, PyObject *exception, const char *format, ...); -PyAPI_FUNC(void) _PyErr_NormalizeException( +extern void _PyErr_NormalizeException( PyThreadState *tstate, PyObject **exc, PyObject **val, PyObject **tb); -PyAPI_FUNC(PyObject *) _PyErr_FormatFromCauseTstate( +extern PyObject* _PyErr_FormatFromCauseTstate( PyThreadState *tstate, PyObject *exception, const char *format, ...); -PyAPI_FUNC(PyObject *) _PyExc_CreateExceptionGroup( +extern PyObject* _PyExc_CreateExceptionGroup( const char *msg, PyObject *excs); -PyAPI_FUNC(PyObject *) _PyExc_PrepReraiseStar( +extern PyObject* _PyExc_PrepReraiseStar( PyObject *orig, PyObject *excs); -PyAPI_FUNC(int) _PyErr_CheckSignalsTstate(PyThreadState *tstate); - -PyAPI_FUNC(void) _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); +extern int _PyErr_CheckSignalsTstate(PyThreadState *tstate); +extern void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); +extern PyObject* _Py_CalculateSuggestions(PyObject *dir, PyObject *name); extern PyObject* _Py_Offer_Suggestions(PyObject* exception); + +// Export for '_testinternalcapi' shared extension PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); void _PyErr_FormatNote(const char *format, ...); +/* Context manipulation (PEP 3134) */ + +Py_DEPRECATED(3.12) extern void _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); + +// Export for '_zoneinfo' shared extension +PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 34dfa53771288e..0ce08900e96f0b 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -1,10 +1,78 @@ -#ifndef Py_INTERNAL_HASH_H -#define Py_INTERNAL_HASH_H +#ifndef Py_INTERNAL_PYHASH_H +#define Py_INTERNAL_PYHASH_H #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif +// Similar to Py_HashPointer(), but don't replace -1 with -2. +static inline Py_hash_t +_Py_HashPointerRaw(const void *ptr) +{ + uintptr_t x = (uintptr_t)ptr; + Py_BUILD_ASSERT(sizeof(x) == sizeof(ptr)); + + // Bottom 3 or 4 bits are likely to be 0; rotate x by 4 to the right + // to avoid excessive hash collisions for dicts and sets. + x = (x >> 4) | (x << (8 * sizeof(uintptr_t) - 4)); + + Py_BUILD_ASSERT(sizeof(x) == sizeof(Py_hash_t)); + return (Py_hash_t)x; +} + +// Export for '_datetime' shared extension +PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); + +/* Hash secret + * + * memory layout on 64 bit systems + * cccccccc cccccccc cccccccc uc -- unsigned char[24] + * pppppppp ssssssss ........ fnv -- two Py_hash_t + * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t + * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t + * ........ ........ eeeeeeee pyexpat XML hash salt + * + * memory layout on 32 bit systems + * cccccccc cccccccc cccccccc uc + * ppppssss ........ ........ fnv -- two Py_hash_t + * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) + * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t + * ........ ........ eeee.... pyexpat XML hash salt + * + * (*) The siphash member may not be available on 32 bit platforms without + * an unsigned int64 data type. + */ +typedef union { + /* ensure 24 bytes */ + unsigned char uc[24]; + /* two Py_hash_t for FNV */ + struct { + Py_hash_t prefix; + Py_hash_t suffix; + } fnv; + /* two uint64 for SipHash24 */ + struct { + uint64_t k0; + uint64_t k1; + } siphash; + /* a different (!) Py_hash_t for small string optimization */ + struct { + unsigned char padding[16]; + Py_hash_t suffix; + } djbx33a; + struct { + unsigned char padding[16]; + Py_hash_t hashsalt; + } expat; +} _Py_HashSecret_t; + +// Export for '_elementtree' shared extension +PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; + +#ifdef Py_DEBUG +extern int _Py_HashSecret_Initialized; +#endif + struct pyhash_runtime_state { struct { @@ -34,7 +102,6 @@ struct pyhash_runtime_state { } -uint64_t _Py_KeyedHash(uint64_t, const char *, Py_ssize_t); - +extern uint64_t _Py_KeyedHash(uint64_t key, const void *src, Py_ssize_t src_sz); -#endif // Py_INTERNAL_HASH_H +#endif // !Py_INTERNAL_PYHASH_H diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 7f8cc643ec0c96..daf7cb77dcc63a 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -23,15 +23,12 @@ extern PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate); extern int _PyUnicode_EnableLegacyWindowsFSEncoding(void); #endif -PyAPI_FUNC(void) _Py_ClearStandardStreamEncoding(void); - -PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc); +extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc); /* Various one-time initializers */ extern void _Py_InitVersion(void); extern PyStatus _PyFaulthandler_Init(int enable); -extern int _PyTraceMalloc_Init(int enable); extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); extern PyStatus _PySys_Create( PyThreadState *tstate, @@ -43,7 +40,6 @@ extern void _PySys_FiniTypes(PyInterpreterState *interp); extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); -extern PyStatus _PyTime_Init(void); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); extern int _Py_Deepfreeze_Init(void); @@ -70,30 +66,56 @@ extern PyStatus _PyGILState_Init(PyInterpreterState *interp); extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); extern void _PyGILState_Fini(PyInterpreterState *interp); -PyAPI_FUNC(void) _PyGC_DumpShutdownStats(PyInterpreterState *interp); +extern void _PyGC_DumpShutdownStats(PyInterpreterState *interp); -PyAPI_FUNC(PyStatus) _Py_PreInitializeFromPyArgv( +extern PyStatus _Py_PreInitializeFromPyArgv( const PyPreConfig *src_config, const struct _PyArgv *args); -PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( +extern PyStatus _Py_PreInitializeFromConfig( const PyConfig *config, const struct _PyArgv *args); -PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void); +extern wchar_t * _Py_GetStdlibDir(void); -PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); +extern int _Py_HandleSystemExit(int *exitcode_p); -PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable); +extern PyObject* _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable); -PyAPI_FUNC(void) _PyErr_Print(PyThreadState *tstate); -PyAPI_FUNC(void) _PyErr_Display(PyObject *file, PyObject *exception, +extern void _PyErr_Print(PyThreadState *tstate); +extern void _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *tb); -PyAPI_FUNC(void) _PyErr_DisplayException(PyObject *file, PyObject *exc); +extern void _PyErr_DisplayException(PyObject *file, PyObject *exc); -PyAPI_FUNC(void) _PyThreadState_DeleteCurrent(PyThreadState *tstate); +extern void _PyThreadState_DeleteCurrent(PyThreadState *tstate); extern void _PyAtExit_Call(PyInterpreterState *interp); +extern int _Py_IsCoreInitialized(void); + +extern int _Py_FdIsInteractive(FILE *fp, PyObject *filename); + +extern const char* _Py_gitidentifier(void); +extern const char* _Py_gitversion(void); + +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp); + +/* Random */ +extern int _PyOS_URandom(void *buffer, Py_ssize_t size); + +// Export for '_random' shared extension +PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size); + +/* Legacy locale support */ +extern int _Py_CoerceLegacyLocale(int warn); +extern int _Py_LegacyLocaleDetected(int warn); + +// Export for 'readline' shared extension +PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); + +// Export for special main.c string compiling with source tracebacks +int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 4cc953d8d779c9..8631ca34a5e616 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_PYMEM_H #define Py_INTERNAL_PYMEM_H + +#include "pycore_lock.h" // PyMutex + #ifdef __cplusplus extern "C" { #endif @@ -8,8 +11,20 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pymem.h" // PyMemAllocatorName +// Try to get the allocators name set by _PyMem_SetupAllocators(). +// Return NULL if unknown. +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(const char*) _PyMem_GetCurrentAllocatorName(void); + +// strdup() using PyMem_RawMalloc() +extern char* _PyMem_RawStrdup(const char *str); + +// strdup() using PyMem_Malloc(). +// Export for '_pickle ' shared extension. +PyAPI_FUNC(char*) _PyMem_Strdup(const char *str); +// wcsdup() using PyMem_RawMalloc() +extern wchar_t* _PyMem_RawWcsdup(const wchar_t *str); typedef struct { /* We tag each block with an API ID in order to tag API violations */ @@ -18,6 +33,7 @@ typedef struct { } debug_alloc_api_t; struct _pymem_allocators { + PyMutex mutex; struct { PyMemAllocatorEx raw; PyMemAllocatorEx mem; @@ -35,7 +51,7 @@ struct _pymem_allocators { /* Set the memory allocator of the specified domain to the default. Save the old allocator into *old_alloc if it's non-NULL. Return on success, or return -1 if the domain is unknown. */ -PyAPI_FUNC(int) _PyMem_SetDefaultAllocator( +extern int _PyMem_SetDefaultAllocator( PyMemAllocatorDomain domain, PyMemAllocatorEx *old_alloc); @@ -81,14 +97,14 @@ static inline int _PyMem_IsPtrFreed(const void *ptr) #endif } -PyAPI_FUNC(int) _PyMem_GetAllocatorName( +extern int _PyMem_GetAllocatorName( const char *name, PyMemAllocatorName *allocator); /* Configure the Python memory allocators. Pass PYMEM_ALLOCATOR_DEFAULT to use default allocators. PYMEM_ALLOCATOR_NOT_SET does nothing. */ -PyAPI_FUNC(int) _PyMem_SetupAllocators(PyMemAllocatorName allocator); +extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator); #ifdef __cplusplus diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 78232738cb09d5..11fbe16fa6f1d5 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_pymem.h" - /********************************/ /* the allocators' initializers */ @@ -20,17 +18,18 @@ extern void * _PyMem_RawRealloc(void *, void *, size_t); extern void _PyMem_RawFree(void *, void *); #define PYRAW_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} -#ifdef WITH_PYMALLOC +#if defined(WITH_PYMALLOC) extern void* _PyObject_Malloc(void *, size_t); extern void* _PyObject_Calloc(void *, size_t, size_t); extern void _PyObject_Free(void *, void *); extern void* _PyObject_Realloc(void *, void *, size_t); # define PYOBJ_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} +# define PYMEM_ALLOC PYOBJ_ALLOC #else # define PYOBJ_ALLOC PYRAW_ALLOC +# define PYMEM_ALLOC PYOBJ_ALLOC #endif // WITH_PYMALLOC -#define PYMEM_ALLOC PYOBJ_ALLOC extern void* _PyMem_DebugRawMalloc(void *, size_t); extern void* _PyMem_DebugRawCalloc(void *, size_t, size_t); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index daa40cf4bcd855..c031a38cd6bfa3 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,7 +8,34 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_runtime.h" /* PyRuntimeState */ +#include "pycore_runtime.h" // _PyRuntime + + +// Values for PyThreadState.state. A thread must be in the "attached" state +// before calling most Python APIs. If the GIL is enabled, then "attached" +// implies that the thread holds the GIL and "detached" implies that the +// thread does not hold the GIL (or is in the process of releasing it). In +// `--disable-gil` builds, multiple threads may be "attached" to the same +// interpreter at the same time. Only the "bound" thread may perform the +// transitions between "attached" and "detached" on its own PyThreadState. +// +// The "gc" state is used to implement stop-the-world pauses, such as for +// cyclic garbage collection. It is only used in `--disable-gil` builds. It is +// similar to the "detached" state, but only the thread performing a +// stop-the-world pause may transition threads between the "detached" and "gc" +// states. A thread trying to "attach" from the "gc" state will block until +// it is transitioned back to "detached" when the stop-the-world pause is +// complete. +// +// State transition diagram: +// +// (bound thread) (stop-the-world thread) +// [attached] <-> [detached] <-> [gc] +// +// See `_PyThreadState_Attach()` and `_PyThreadState_Detach()`. +#define _Py_THREAD_DETACHED 0 +#define _Py_THREAD_ATTACHED 1 +#define _Py_THREAD_GC 2 /* Check if the current thread is the main thread. @@ -36,10 +63,20 @@ _Py_IsMainInterpreter(PyInterpreterState *interp) static inline int _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) { - return (_PyRuntimeState_GetFinalizing(interp->runtime) != NULL && - interp == &interp->runtime->_main_interpreter); + /* bpo-39877: Access _PyRuntime directly rather than using + tstate->interp->runtime to support calls from Python daemon threads. + After Py_Finalize() has been called, tstate can be a dangling pointer: + point to PyThreadState freed memory. */ + return (_PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL && + interp == &_PyRuntime._main_interpreter); } +// Export for _xxsubinterpreters module. +PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_FailIfRunningMain(PyInterpreterState *); + static inline const PyConfig * _Py_GetMainConfig(void) @@ -60,21 +97,22 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) } -/* Only execute pending calls on the main thread. */ -static inline int -_Py_ThreadCanHandlePendingCalls(void) -{ - return _Py_IsMainThread(); -} - - /* Variable and static inline functions for in-line access to current thread and interpreter state */ #if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) extern _Py_thread_local PyThreadState *_Py_tss_tstate; #endif -PyAPI_DATA(PyThreadState *) _PyThreadState_GetCurrent(void); + +#ifndef NDEBUG +extern int _PyThreadState_CheckConsistency(PyThreadState *tstate); +#endif + +int _PyThreadState_MustExit(PyThreadState *tstate); + +// Export for most shared extensions, used via _PyThreadState_GET() static +// inline function. +PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); /* Get the current Python thread state. @@ -82,7 +120,7 @@ PyAPI_DATA(PyThreadState *) _PyThreadState_GetCurrent(void); The caller must hold the GIL. - See also PyThreadState_Get() and _PyThreadState_UncheckedGet(). */ + See also PyThreadState_Get() and PyThreadState_GetUnchecked(). */ static inline PyThreadState* _PyThreadState_GET(void) { @@ -93,6 +131,21 @@ _PyThreadState_GET(void) #endif } +// Attaches the current thread to the interpreter. +// +// This may block while acquiring the GIL (if the GIL is enabled) or while +// waiting for a stop-the-world pause (if the GIL is disabled). +// +// High-level code should generally call PyEval_RestoreThread() instead, which +// calls this function. +void _PyThreadState_Attach(PyThreadState *tstate); + +// Detaches the current thread from the interpreter. +// +// High-level code should generally call PyEval_SaveThread() instead, which +// calls this function. +void _PyThreadState_Detach(PyThreadState *tstate); + static inline void _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) @@ -116,7 +169,7 @@ _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) The caller must hold the GIL. - See also _PyInterpreterState_Get() + See also PyInterpreterState_Get() and _PyGILState_GetInterpreterStateUnsafe(). */ static inline PyInterpreterState* _PyInterpreterState_GET(void) { PyThreadState *tstate = _PyThreadState_GET(); @@ -129,46 +182,61 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) { // PyThreadState functions -PyAPI_FUNC(PyThreadState *) _PyThreadState_New(PyInterpreterState *interp); -PyAPI_FUNC(void) _PyThreadState_Bind(PyThreadState *tstate); -// We keep this around exclusively for stable ABI compatibility. -PyAPI_FUNC(void) _PyThreadState_Init( - PyThreadState *tstate); -PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate); +extern PyThreadState * _PyThreadState_New( + PyInterpreterState *interp, + int whence); +extern void _PyThreadState_Bind(PyThreadState *tstate); +extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); -extern void _PyThreadState_InitDetached(PyThreadState *, PyInterpreterState *); -extern void _PyThreadState_ClearDetached(PyThreadState *); -extern void _PyThreadState_BindDetached(PyThreadState *); -extern void _PyThreadState_UnbindDetached(PyThreadState *); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate); + +/* The implementation of sys._current_exceptions() Returns a dict mapping + thread id to that thread's current exception. +*/ +extern PyObject* _PyThread_CurrentExceptions(void); /* Other */ -PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( +extern PyThreadState * _PyThreadState_Swap( _PyRuntimeState *runtime, PyThreadState *newts); -PyAPI_FUNC(PyStatus) _PyInterpreterState_Enable(_PyRuntimeState *runtime); +extern PyStatus _PyInterpreterState_Enable(_PyRuntimeState *runtime); #ifdef HAVE_FORK extern PyStatus _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime); extern void _PySignal_AfterFork(void); #endif - +// Export for the stable ABI PyAPI_FUNC(int) _PyState_AddModule( PyThreadState *tstate, PyObject* module, PyModuleDef* def); -PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); +extern int _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_LOCK(runtime) \ - PyThread_acquire_lock((runtime)->interpreters.mutex, WAIT_LOCK) + PyMutex_LockFlags(&(runtime)->interpreters.mutex, _Py_LOCK_DONT_DETACH) #define HEAD_UNLOCK(runtime) \ - PyThread_release_lock((runtime)->interpreters.mutex) - + PyMutex_Unlock(&(runtime)->interpreters.mutex) + +// Get the configuration of the current interpreter. +// The caller must hold the GIL. +// Export for test_peg_generator. +PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); + +// Get the single PyInterpreterState used by this process' GILState +// implementation. +// +// This function doesn't check for error. Return NULL before _PyGILState_Init() +// is called and after _PyGILState_Fini() is called. +// +// See also PyInterpreterState_Get() and _PyInterpreterState_GET(). +extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pystats.h b/Include/internal/pycore_pystats.h new file mode 100644 index 00000000000000..f8af398a560586 --- /dev/null +++ b/Include/internal/pycore_pystats.h @@ -0,0 +1,21 @@ +#ifndef Py_INTERNAL_PYSTATS_H +#define Py_INTERNAL_PYSTATS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_STATS +extern void _Py_StatsOn(void); +extern void _Py_StatsOff(void); +extern void _Py_StatsClear(void); +extern int _Py_PrintSpecializationStats(int to_file); +#endif + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_PYSTATS_H diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h new file mode 100644 index 00000000000000..0bfc5704dc4c59 --- /dev/null +++ b/Include/internal/pycore_pythonrun.h @@ -0,0 +1,39 @@ +#ifndef Py_INTERNAL_PYTHONRUN_H +#define Py_INTERNAL_PYTHONRUN_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern int _PyRun_SimpleFileObject( + FILE *fp, + PyObject *filename, + int closeit, + PyCompilerFlags *flags); + +extern int _PyRun_AnyFileObject( + FILE *fp, + PyObject *filename, + int closeit, + PyCompilerFlags *flags); + +extern int _PyRun_InteractiveLoopObject( + FILE *fp, + PyObject *filename, + PyCompilerFlags *flags); + +extern const char* _Py_SourceAsString( + PyObject *cmd, + const char *funcname, + const char *what, + PyCompilerFlags *cf, + PyObject **cmd_copy); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_PYTHONRUN_H + diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index f53921494c158f..9c9a09f60f3441 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -8,42 +8,45 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "dynamic_annotations.h" // _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX + +// Get _POSIX_THREADS and _POSIX_SEMAPHORES macros if available +#if (defined(HAVE_UNISTD_H) && !defined(_POSIX_THREADS) \ + && !defined(_POSIX_SEMAPHORES)) +# include // _POSIX_THREADS, _POSIX_SEMAPHORES +#endif +#if (defined(HAVE_PTHREAD_H) && !defined(_POSIX_THREADS) \ + && !defined(_POSIX_SEMAPHORES)) + // This means pthreads are not implemented in libc headers, hence the macro + // not present in . But they still can be implemented as an + // external library (e.g. gnu pth in pthread emulation) +# include // _POSIX_THREADS, _POSIX_SEMAPHORES +#endif +#if !defined(_POSIX_THREADS) && defined(__hpux) && defined(_SC_THREADS) + // Check if we're running on HP-UX and _SC_THREADS is defined. If so, then + // enough of the POSIX threads package is implemented to support Python + // threads. + // + // This is valid for HP-UX 11.23 running on an ia64 system. If needed, add + // a check of __ia64 to verify that we're running on an ia64 system instead + // of a pa-risc system. +# define _POSIX_THREADS +#endif -#ifndef _POSIX_THREADS -/* This means pthreads are not implemented in libc headers, hence the macro - not present in unistd.h. But they still can be implemented as an external - library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include /* _POSIX_THREADS */ -# endif -# ifndef _POSIX_THREADS -/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then - enough of the Posix threads package is implemented to support python - threads. - - This is valid for HP-UX 11.23 running on an ia64 system. If needed, add - a check of __ia64 to verify that we're running on an ia64 system instead - of a pa-risc system. -*/ -# ifdef __hpux -# ifdef _SC_THREADS -# define _POSIX_THREADS -# endif -# endif -# endif /* _POSIX_THREADS */ -#endif /* _POSIX_THREADS */ #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS) -# define _USE_PTHREADS +# define _USE_PTHREADS #endif #if defined(_USE_PTHREADS) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) // monotonic is supported statically. It doesn't mean it works on runtime. -# define CONDATTR_MONOTONIC +# define CONDATTR_MONOTONIC #endif #if defined(HAVE_PTHREAD_STUBS) +#include // bool + // pthread_key struct py_stub_tls_entry { bool in_use; @@ -75,6 +78,77 @@ struct _pythread_runtime_state { }; +#ifdef HAVE_FORK +/* Private function to reinitialize a lock at fork in the child process. + Reset the lock to the unlocked state. + Return 0 on success, return -1 on error. */ +extern int _PyThread_at_fork_reinit(PyThread_type_lock *lock); +#endif /* HAVE_FORK */ + + +// unset: -1 seconds, in nanoseconds +#define PyThread_UNSET_TIMEOUT ((_PyTime_t)(-1 * 1000 * 1000 * 1000)) + +// Exported for the _xxinterpchannels module. +PyAPI_FUNC(int) PyThread_ParseTimeoutArg( + PyObject *arg, + int blocking, + PY_TIMEOUT_T *timeout); + +/* Helper to acquire an interruptible lock with a timeout. If the lock acquire + * is interrupted, signal handlers are run, and if they raise an exception, + * PY_LOCK_INTR is returned. Otherwise, PY_LOCK_ACQUIRED or PY_LOCK_FAILURE + * are returned, depending on whether the lock can be acquired within the + * timeout. + */ +// Exported for the _xxinterpchannels module. +PyAPI_FUNC(PyLockStatus) PyThread_acquire_lock_timed_with_retries( + PyThread_type_lock, + PY_TIMEOUT_T microseconds); + +typedef unsigned long long PyThread_ident_t; +typedef Py_uintptr_t PyThread_handle_t; + +#define PY_FORMAT_THREAD_IDENT_T "llu" +#define Py_PARSE_THREAD_IDENT_T "K" + +PyAPI_FUNC(PyThread_ident_t) PyThread_get_thread_ident_ex(void); + +/* Thread joining APIs. + * + * These APIs have a strict contract: + * - Either PyThread_join_thread or PyThread_detach_thread must be called + * exactly once with the given handle. + * - Calling neither PyThread_join_thread nor PyThread_detach_thread results + * in a resource leak until the end of the process. + * - Any other usage, such as calling both PyThread_join_thread and + * PyThread_detach_thread, or calling them more than once (including + * simultaneously), results in undefined behavior. + */ +PyAPI_FUNC(int) PyThread_start_joinable_thread(void (*func)(void *), + void *arg, + PyThread_ident_t* ident, + PyThread_handle_t* handle); +/* + * Join a thread started with `PyThread_start_joinable_thread`. + * This function cannot be interrupted. It returns 0 on success, + * a non-zero value on failure. + */ +PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); +/* + * Detach a thread started with `PyThread_start_joinable_thread`, such + * that its resources are relased as soon as it exits. + * This function cannot be interrupted. It returns 0 on success, + * a non-zero value on failure. + */ +PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); + +/* + * Obtain the new thread ident and handle in a forked child process. + */ +PyAPI_FUNC(void) PyThread_update_thread_after_fork(PyThread_ident_t* ident, + PyThread_handle_t* handle); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 6e06e874711bc2..e3348296ea61b7 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -8,27 +8,24 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atexit.h" // struct atexit_runtime_state -#include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_atexit.h" // struct _atexit_runtime_state #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_floatobject.h" // struct _Py_float_runtime_state +#include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state -#include "pycore_global_objects.h" // struct _Py_global_objects +#include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState #include "pycore_object_state.h" // struct _py_object_runtime_state #include "pycore_parser.h" // struct _parser_runtime_state -#include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state +#include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_signal.h" // struct _signals_runtime_state -#include "pycore_time.h" // struct _time_runtime_state #include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state -#include "pycore_typeobject.h" // struct types_runtime_state -#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids +#include "pycore_typeobject.h" // struct _types_runtime_state +#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state struct _getargs_runtime_state { - PyThread_type_lock mutex; struct _PyArg_Parser *static_parsers; }; @@ -53,12 +50,101 @@ typedef struct _Py_AuditHookEntry { void *userData; } _Py_AuditHookEntry; +typedef struct _Py_DebugOffsets { + char cookie[8]; + uint64_t version; + // Runtime state offset; + struct _runtime_state { + off_t finalizing; + off_t interpreters_head; + } runtime_state; + + // Interpreter state offset; + struct _interpreter_state { + off_t next; + off_t threads_head; + off_t gc; + off_t imports_modules; + off_t sysdict; + off_t builtins; + off_t ceval_gil; + off_t gil_runtime_state_locked; + off_t gil_runtime_state_holder; + } interpreter_state; + + // Thread state offset; + struct _thread_state{ + off_t prev; + off_t next; + off_t interp; + off_t current_frame; + off_t thread_id; + off_t native_thread_id; + } thread_state; + + // InterpreterFrame offset; + struct _interpreter_frame { + off_t previous; + off_t executable; + off_t instr_ptr; + off_t localsplus; + off_t owner; + } interpreter_frame; + + // CFrame offset; + struct _cframe { + off_t current_frame; + off_t previous; + } cframe; + + // Code object offset; + struct _code_object { + off_t filename; + off_t name; + off_t linetable; + off_t firstlineno; + off_t argcount; + off_t localsplusnames; + off_t localspluskinds; + off_t co_code_adaptive; + } code_object; + + // PyObject offset; + struct _pyobject { + off_t ob_type; + } pyobject; + + // PyTypeObject object offset; + struct _type_object { + off_t tp_name; + } type_object; + + // PyTuple object offset; + struct _tuple_object { + off_t ob_item; + } tuple_object; +} _Py_DebugOffsets; + /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. That data is exposed in the internal API as a static variable (_PyRuntime). */ typedef struct pyruntimestate { + /* This field must be first to facilitate locating it by out of process + * debuggers. Out of process debuggers will use the offsets contained in this + * field to be able to locate other fields in several interpreter structures + * in a way that doesn't require them to know the exact layout of those + * structures. + * + * IMPORTANT: + * This struct is **NOT** backwards compatible between minor version of the + * interpreter and the members, order of members and size can change between + * minor versions. This struct is only guaranteed to be stable between patch + * versions for a given minor version of the interpreter. + */ + _Py_DebugOffsets debug_offsets; + /* Has been initialized to a safe state. In order to be effective, this must be set to 0 during or right @@ -82,17 +168,12 @@ typedef struct pyruntimestate { Use _PyRuntimeState_GetFinalizing() and _PyRuntimeState_SetFinalizing() to access it, don't access it directly. */ - _Py_atomic_address _finalizing; - - struct _pymem_allocators allocators; - struct _obmalloc_global_state obmalloc; - struct pyhash_runtime_state pyhash_state; - struct _time_runtime_state time; - struct _pythread_runtime_state threads; - struct _signals_runtime_state signals; + PyThreadState *_finalizing; + /* The ID of the OS thread in which we are finalizing. */ + unsigned long _finalizing_id; struct pyinterpreters { - PyThread_type_lock mutex; + PyMutex mutex; /* The linked list of interpreters, newest first. */ PyInterpreterState *head; /* The runtime's initial interpreter, which has a special role @@ -109,14 +190,23 @@ typedef struct pyruntimestate { using a Python int. */ int64_t next_id; } interpreters; - // XXX Remove this field once we have a tp_* slot. - struct _xidregistry { - PyThread_type_lock mutex; - struct _xidregitem *head; - } xidregistry; unsigned long main_thread; + /* ---------- IMPORTANT --------------------------- + The fields above this line are declared as early as + possible to facilitate out-of-process observability + tools. */ + + /* cross-interpreter data and utils */ + struct _xi_runtime_state xi; + + struct _pymem_allocators allocators; + struct _obmalloc_global_state obmalloc; + struct pyhash_runtime_state pyhash_state; + struct _pythread_runtime_state threads; + struct _signals_runtime_state signals; + /* Used for the thread state bound to the current thread. */ Py_tss_t autoTSSkey; @@ -143,7 +233,10 @@ typedef struct pyruntimestate { // is called multiple times. Py_OpenCodeHookFunction open_code_hook; void *open_code_userdata; - _Py_AuditHookEntry *audit_hook_head; + struct { + PyMutex mutex; + _Py_AuditHookEntry *head; + } audit_hooks; struct _py_object_runtime_state object_state; struct _Py_float_runtime_state float_state; @@ -151,6 +244,7 @@ typedef struct pyruntimestate { struct _types_runtime_state types; /* All the objects that are shared by the runtime's interpreters. */ + struct _Py_cached_objects cached_objects; struct _Py_static_objects static_objects; /* The following fields are here to avoid allocation during init. @@ -169,15 +263,25 @@ typedef struct pyruntimestate { /* PyInterpreterState.interpreters.main */ PyInterpreterState _main_interpreter; + +#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) + // Used in "Python/emscripten_trampoline.c" to choose between type + // reflection trampoline and EM_JS trampoline. + bool wasm_type_reflection_available; +#endif + } _PyRuntimeState; /* other API */ +// Export _PyRuntime for shared extensions which use it in static inline +// functions for best performance, like _Py_IsMainThread() or _Py_ID(). +// It's also made accessible for debuggers and profilers. PyAPI_DATA(_PyRuntimeState) _PyRuntime; -PyAPI_FUNC(PyStatus) _PyRuntimeState_Init(_PyRuntimeState *runtime); -PyAPI_FUNC(void) _PyRuntimeState_Fini(_PyRuntimeState *runtime); +extern PyStatus _PyRuntimeState_Init(_PyRuntimeState *runtime); +extern void _PyRuntimeState_Fini(_PyRuntimeState *runtime); #ifdef HAVE_FORK extern PyStatus _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime); @@ -185,19 +289,33 @@ extern PyStatus _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime); /* Initialize _PyRuntimeState. Return NULL on success, or return an error message on failure. */ -PyAPI_FUNC(PyStatus) _PyRuntime_Initialize(void); +extern PyStatus _PyRuntime_Initialize(void); -PyAPI_FUNC(void) _PyRuntime_Finalize(void); +extern void _PyRuntime_Finalize(void); static inline PyThreadState* _PyRuntimeState_GetFinalizing(_PyRuntimeState *runtime) { - return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->_finalizing); + return (PyThreadState*)_Py_atomic_load_ptr_relaxed(&runtime->_finalizing); +} + +static inline unsigned long +_PyRuntimeState_GetFinalizingID(_PyRuntimeState *runtime) { + return _Py_atomic_load_ulong_relaxed(&runtime->_finalizing_id); } static inline void _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { - _Py_atomic_store_relaxed(&runtime->_finalizing, (uintptr_t)tstate); + _Py_atomic_store_ptr_relaxed(&runtime->_finalizing, tstate); + if (tstate == NULL) { + _Py_atomic_store_ulong_relaxed(&runtime->_finalizing_id, 0); + } + else { + // XXX Re-enable this assert once gh-109860 is fixed. + //assert(tstate->thread_id == PyThread_get_thread_ident()); + _Py_atomic_store_ulong_relaxed(&runtime->_finalizing_id, + tstate->thread_id); + } } #ifdef __cplusplus diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 3b1444f3429b9a..d324a94278839c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -8,11 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_long.h" -#include "pycore_object.h" -#include "pycore_parser.h" -#include "pycore_pymem_init.h" -#include "pycore_obmalloc_init.h" +#include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT +#include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT +#include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_object.h" // _PyObject_HEAD_INIT +#include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT +#include "pycore_parser.h" // _parser_runtime_state_INIT +#include "pycore_pyhash.h" // pyhash_state_INIT +#include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT +#include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT +#include "pycore_signal.h" // _signals_RUNTIME_INIT +#include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT extern PyTypeObject _PyExc_MemoryError; @@ -21,13 +27,65 @@ extern PyTypeObject _PyExc_MemoryError; /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ - #define _PyRuntimeState_INIT(runtime) \ { \ + .debug_offsets = { \ + .cookie = "xdebugpy", \ + .version = PY_VERSION_HEX, \ + .runtime_state = { \ + .finalizing = offsetof(_PyRuntimeState, _finalizing), \ + .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ + }, \ + .interpreter_state = { \ + .next = offsetof(PyInterpreterState, next), \ + .threads_head = offsetof(PyInterpreterState, threads.head), \ + .gc = offsetof(PyInterpreterState, gc), \ + .imports_modules = offsetof(PyInterpreterState, imports.modules), \ + .sysdict = offsetof(PyInterpreterState, sysdict), \ + .builtins = offsetof(PyInterpreterState, builtins), \ + .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ + .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + }, \ + .thread_state = { \ + .prev = offsetof(PyThreadState, prev), \ + .next = offsetof(PyThreadState, next), \ + .interp = offsetof(PyThreadState, interp), \ + .current_frame = offsetof(PyThreadState, current_frame), \ + .thread_id = offsetof(PyThreadState, thread_id), \ + .native_thread_id = offsetof(PyThreadState, native_thread_id), \ + }, \ + .interpreter_frame = { \ + .previous = offsetof(_PyInterpreterFrame, previous), \ + .executable = offsetof(_PyInterpreterFrame, f_executable), \ + .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ + .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ + .owner = offsetof(_PyInterpreterFrame, owner), \ + }, \ + .code_object = { \ + .filename = offsetof(PyCodeObject, co_filename), \ + .name = offsetof(PyCodeObject, co_name), \ + .linetable = offsetof(PyCodeObject, co_linetable), \ + .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ + .argcount = offsetof(PyCodeObject, co_argcount), \ + .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ + .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ + .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + }, \ + .pyobject = { \ + .ob_type = offsetof(PyObject, ob_type), \ + }, \ + .type_object = { \ + .tp_name = offsetof(PyTypeObject, tp_name), \ + }, \ + .tuple_object = { \ + .ob_item = offsetof(PyTupleObject, ob_item), \ + }, \ + }, \ .allocators = { \ - _pymem_allocators_standard_INIT(runtime), \ - _pymem_allocators_debug_INIT, \ - _pymem_allocators_obj_arena_INIT, \ + .standard = _pymem_allocators_standard_INIT(runtime), \ + .debug = _pymem_allocators_debug_INIT, \ + .obj_arena = _pymem_allocators_obj_arena_INIT, \ }, \ .obmalloc = _obmalloc_global_state_INIT, \ .pyhash_state = pyhash_state_INIT, \ @@ -37,15 +95,15 @@ extern PyTypeObject _PyExc_MemoryError; until _PyInterpreterState_Enable() is called. */ \ .next_id = -1, \ }, \ + .xi = { \ + .registry = { \ + .global = 1, \ + }, \ + }, \ /* A TSS key must be initialized with Py_tss_NEEDS_INIT \ in accordance with the specification. */ \ .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ - .imports = { \ - .extensions = { \ - .main_tstate = _PyThreadState_INIT, \ - }, \ - }, \ .ceval = { \ .perf = _PyEval_RUNTIME_PERF_INIT, \ }, \ @@ -76,13 +134,13 @@ extern PyTypeObject _PyExc_MemoryError; .latin1 = _Py_str_latin1_INIT, \ }, \ .tuple_empty = { \ - .ob_base = _PyVarObject_HEAD_INIT(&PyTuple_Type, 0) \ + .ob_base = _PyVarObject_HEAD_INIT(&PyTuple_Type, 0), \ }, \ .hamt_bitmap_node_empty = { \ - .ob_base = _PyVarObject_HEAD_INIT(&_PyHamt_BitmapNode_Type, 0) \ + .ob_base = _PyVarObject_HEAD_INIT(&_PyHamt_BitmapNode_Type, 0), \ }, \ .context_token_missing = { \ - .ob_base = _PyObject_HEAD_INIT(&_PyContextTokenMissing_Type) \ + .ob_base = _PyObject_HEAD_INIT(&_PyContextTokenMissing_Type), \ }, \ }, \ }, \ @@ -106,6 +164,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ + .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ .func_state = { \ @@ -118,29 +177,46 @@ extern PyTypeObject _PyExc_MemoryError; .singletons = { \ ._not_used = 1, \ .hamt_empty = { \ - .ob_base = _PyObject_HEAD_INIT(&_PyHamt_Type) \ + .ob_base = _PyObject_HEAD_INIT(&_PyHamt_Type), \ .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \ }, \ .last_resort_memory_error = { \ - _PyObject_HEAD_INIT(&_PyExc_MemoryError) \ + _PyObject_HEAD_INIT(&_PyExc_MemoryError), \ + .args = (PyObject*)&_Py_SINGLETON(tuple_empty) \ }, \ }, \ }, \ - ._initial_thread = _PyThreadState_INIT, \ + ._initial_thread = _PyThreadStateImpl_INIT, \ + } + +#define _PyThreadStateImpl_INIT \ + { \ + .base = _PyThreadState_INIT, \ } #define _PyThreadState_INIT \ { \ + ._whence = _PyThreadState_WHENCE_NOTSET, \ .py_recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ .context_ver = 1, \ } +#ifdef Py_TRACE_REFS +# define _py_object_state_INIT(INTERP) \ + { \ + .refchain = NULL, \ + } +#else +# define _py_object_state_INIT(INTERP) \ + { 0 } +#endif + // global objects #define _PyBytes_SIMPLE_INIT(CH, LEN) \ { \ - _PyVarObject_HEAD_INIT(&PyBytes_Type, (LEN)) \ + _PyVarObject_HEAD_INIT(&PyBytes_Type, (LEN)), \ .ob_shash = -1, \ .ob_sval = { (CH) }, \ } @@ -151,13 +227,14 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyUnicode_ASCII_BASE_INIT(LITERAL, ASCII) \ { \ - .ob_base = _PyObject_HEAD_INIT(&PyUnicode_Type) \ + .ob_base = _PyObject_HEAD_INIT(&PyUnicode_Type), \ .length = sizeof(LITERAL) - 1, \ .hash = -1, \ .state = { \ .kind = 1, \ .compact = 1, \ .ascii = (ASCII), \ + .statically_allocated = 1, \ }, \ } #define _PyASCIIObject_INIT(LITERAL) \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 7b9c73dd1edf3b..1defa39f816e78 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -8,6 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_long.h" // _PyLong_DIGIT_INIT() + + /* The following is auto-generated by Tools/build/generate_global_objects.py. */ #define _Py_small_ints_INIT { \ _PyLong_DIGIT_INIT(-5), \ @@ -539,6 +542,7 @@ extern "C" { INIT_STR(anon_lambda, ""), \ INIT_STR(anon_listcomp, ""), \ INIT_STR(anon_module, ""), \ + INIT_STR(anon_null, ""), \ INIT_STR(anon_setcomp, ""), \ INIT_STR(anon_string, ""), \ INIT_STR(anon_unknown, ""), \ @@ -546,15 +550,18 @@ extern "C" { INIT_STR(dbl_close_br, "}}"), \ INIT_STR(dbl_open_br, "{{"), \ INIT_STR(dbl_percent, "%%"), \ + INIT_STR(defaults, ".defaults"), \ INIT_STR(dot, "."), \ INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ + INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ + INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ INIT_STR(newline, "\n"), \ INIT_STR(open_br, "{"), \ INIT_STR(percent, "%"), \ - INIT_STR(shim_name, ""), \ + INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } @@ -596,6 +603,8 @@ extern "C" { INIT_ID(__class__), \ INIT_ID(__class_getitem__), \ INIT_ID(__classcell__), \ + INIT_ID(__classdict__), \ + INIT_ID(__classdictcell__), \ INIT_ID(__complex__), \ INIT_ID(__contains__), \ INIT_ID(__copy__), \ @@ -658,6 +667,7 @@ extern "C" { INIT_ID(__lshift__), \ INIT_ID(__lt__), \ INIT_ID(__main__), \ + INIT_ID(__match_args__), \ INIT_ID(__matmul__), \ INIT_ID(__missing__), \ INIT_ID(__mod__), \ @@ -718,6 +728,7 @@ extern "C" { INIT_ID(__subclasshook__), \ INIT_ID(__truediv__), \ INIT_ID(__trunc__), \ + INIT_ID(__type_params__), \ INIT_ID(__typing_is_unpacked_typevartuple__), \ INIT_ID(__typing_prepare_subst__), \ INIT_ID(__typing_subst__), \ @@ -768,6 +779,7 @@ extern "C" { INIT_ID(a), \ INIT_ID(abs_tol), \ INIT_ID(access), \ + INIT_ID(aclose), \ INIT_ID(add), \ INIT_ID(add_done_callback), \ INIT_ID(after_in_child), \ @@ -778,7 +790,9 @@ extern "C" { INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ + INIT_ID(asend), \ INIT_ID(ast), \ + INIT_ID(athrow), \ INIT_ID(attribute), \ INIT_ID(authorizer_callback), \ INIT_ID(autocommit), \ @@ -789,6 +803,7 @@ extern "C" { INIT_ID(big), \ INIT_ID(binary_form), \ INIT_ID(block), \ + INIT_ID(bound), \ INIT_ID(buffer), \ INIT_ID(buffer_callback), \ INIT_ID(buffer_size), \ @@ -809,6 +824,7 @@ extern "C" { INIT_ID(call), \ INIT_ID(call_exception_handler), \ INIT_ID(call_soon), \ + INIT_ID(callback), \ INIT_ID(cancel), \ INIT_ID(capath), \ INIT_ID(category), \ @@ -844,11 +860,13 @@ extern "C" { INIT_ID(compile_mode), \ INIT_ID(consts), \ INIT_ID(context), \ + INIT_ID(contravariant), \ INIT_ID(cookie), \ INIT_ID(copy), \ INIT_ID(copyreg), \ INIT_ID(coro), \ INIT_ID(count), \ + INIT_ID(covariant), \ INIT_ID(cwd), \ INIT_ID(d), \ INIT_ID(data), \ @@ -877,7 +895,6 @@ extern "C" { INIT_ID(dont_inherit), \ INIT_ID(dst), \ INIT_ID(dst_dir_fd), \ - INIT_ID(duration), \ INIT_ID(e), \ INIT_ID(eager_start), \ INIT_ID(effective_ids), \ @@ -893,12 +910,13 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ - INIT_ID(exc_type), \ - INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ + INIT_ID(existing_file_name), \ INIT_ID(exp), \ INIT_ID(extend), \ + INIT_ID(extra_tokens), \ + INIT_ID(f), \ INIT_ID(facility), \ INIT_ID(factory), \ INIT_ID(false), \ @@ -922,7 +940,6 @@ extern "C" { INIT_ID(flush), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ - INIT_ID(frequency), \ INIT_ID(from_param), \ INIT_ID(fromlist), \ INIT_ID(fromtimestamp), \ @@ -930,6 +947,7 @@ extern "C" { INIT_ID(fset), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(g), \ INIT_ID(generation), \ INIT_ID(genexpr), \ INIT_ID(get), \ @@ -943,6 +961,7 @@ extern "C" { INIT_ID(globals), \ INIT_ID(groupindex), \ INIT_ID(groups), \ + INIT_ID(h), \ INIT_ID(handle), \ INIT_ID(hash_name), \ INIT_ID(header), \ @@ -951,6 +970,7 @@ extern "C" { INIT_ID(hook), \ INIT_ID(id), \ INIT_ID(ident), \ + INIT_ID(identity_hint), \ INIT_ID(ignore), \ INIT_ID(imag), \ INIT_ID(importlib), \ @@ -958,6 +978,7 @@ extern "C" { INIT_ID(incoming), \ INIT_ID(indexgroup), \ INIT_ID(inf), \ + INIT_ID(infer_variance), \ INIT_ID(inheritable), \ INIT_ID(initial), \ INIT_ID(initial_bytes), \ @@ -970,6 +991,7 @@ extern "C" { INIT_ID(instructions), \ INIT_ID(intern), \ INIT_ID(intersection), \ + INIT_ID(interval), \ INIT_ID(is_running), \ INIT_ID(isatty), \ INIT_ID(isinstance), \ @@ -1047,10 +1069,12 @@ extern "C" { INIT_ID(namespaces), \ INIT_ID(narg), \ INIT_ID(ndigits), \ + INIT_ID(new_file_name), \ INIT_ID(new_limit), \ INIT_ID(newline), \ INIT_ID(newlines), \ INIT_ID(next), \ + INIT_ID(nlocals), \ INIT_ID(node_depth), \ INIT_ID(node_offset), \ INIT_ID(ns), \ @@ -1074,6 +1098,7 @@ extern "C" { INIT_ID(optimize), \ INIT_ID(options), \ INIT_ID(order), \ + INIT_ID(origin), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ INIT_ID(overlapped), \ @@ -1099,6 +1124,7 @@ extern "C" { INIT_ID(priority), \ INIT_ID(progress), \ INIT_ID(progress_handler), \ + INIT_ID(progress_routine), \ INIT_ID(proto), \ INIT_ID(protocol), \ INIT_ID(ps1), \ @@ -1158,7 +1184,6 @@ extern "C" { INIT_ID(sleep), \ INIT_ID(sock), \ INIT_ID(sort), \ - INIT_ID(sound), \ INIT_ID(source), \ INIT_ID(source_traceback), \ INIT_ID(src), \ @@ -1171,6 +1196,7 @@ extern "C" { INIT_ID(stdin), \ INIT_ID(stdout), \ INIT_ID(step), \ + INIT_ID(steps), \ INIT_ID(store_name), \ INIT_ID(strategy), \ INIT_ID(strftime), \ @@ -1199,7 +1225,6 @@ extern "C" { INIT_ID(timetuple), \ INIT_ID(top), \ INIT_ID(trace_callback), \ - INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ INIT_ID(true), \ @@ -1207,6 +1232,7 @@ extern "C" { INIT_ID(twice), \ INIT_ID(txt), \ INIT_ID(type), \ + INIT_ID(type_params), \ INIT_ID(tz), \ INIT_ID(tzname), \ INIT_ID(uid), \ diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h new file mode 100644 index 00000000000000..4c37df7b39a48a --- /dev/null +++ b/Include/internal/pycore_semaphore.h @@ -0,0 +1,66 @@ +// The _PySemaphore API a simplified cross-platform semaphore used to implement +// wakeup/sleep. +#ifndef Py_INTERNAL_SEMAPHORE_H +#define Py_INTERNAL_SEMAPHORE_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_pythread.h" // _POSIX_SEMAPHORES +#include "pycore_time.h" // _PyTime_t + +#ifdef MS_WINDOWS +# define WIN32_LEAN_AND_MEAN +# include +#elif defined(HAVE_PTHREAD_H) +# include +#elif defined(HAVE_PTHREAD_STUBS) +# include "cpython/pthread_stubs.h" +#else +# error "Require native threads. See https://bugs.python.org/issue31370" +#endif + +#if (defined(_POSIX_SEMAPHORES) && (_POSIX_SEMAPHORES+0) != -1 && \ + defined(HAVE_SEM_TIMEDWAIT)) +# define _Py_USE_SEMAPHORES +# include +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _PySemaphore { +#if defined(MS_WINDOWS) + HANDLE platform_sem; +#elif defined(_Py_USE_SEMAPHORES) + sem_t platform_sem; +#else + pthread_mutex_t mutex; + pthread_cond_t cond; + int counter; +#endif +} _PySemaphore; + +// Puts the current thread to sleep until _PySemaphore_Wakeup() is called. +// If `detach` is true, then the thread will detach/release the GIL while +// sleeping. +PyAPI_FUNC(int) +_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout_ns, int detach); + +// Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() +// can be called before _PySemaphore_Wait(). +PyAPI_FUNC(void) +_PySemaphore_Wakeup(_PySemaphore *sema); + +// Initializes/destroys a semaphore +PyAPI_FUNC(void) _PySemaphore_Init(_PySemaphore *sema); +PyAPI_FUNC(void) _PySemaphore_Destroy(_PySemaphore *sema); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_SEMAPHORE_H */ diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h new file mode 100644 index 00000000000000..34a00e6d45fe69 --- /dev/null +++ b/Include/internal/pycore_setobject.h @@ -0,0 +1,27 @@ +#ifndef Py_INTERNAL_SETOBJECT_H +#define Py_INTERNAL_SETOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Export for '_pickle' shared extension +PyAPI_FUNC(int) _PySet_NextEntry( + PyObject *set, + Py_ssize_t *pos, + PyObject **key, + Py_hash_t *hash); + +// Export for '_pickle' shared extension +PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); + +// Export for the gdb plugin's (python-gdb.py) benefit +PyAPI_DATA(PyObject *) _PySet_Dummy; + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_SETOBJECT_H diff --git a/Include/internal/pycore_signal.h b/Include/internal/pycore_signal.h index ca3f69d09fc0c1..47213a34ab77b5 100644 --- a/Include/internal/pycore_signal.h +++ b/Include/internal/pycore_signal.h @@ -10,10 +10,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" // _Py_atomic_address +#include // NSIG -#include // NSIG +// Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. +// Export for '_posixsubprocess' shared extension. +PyAPI_FUNC(void) _Py_RestoreSignals(void); #ifdef _SIG_MAXSIG // gh-91145: On FreeBSD, defines NSIG as 32: it doesn't include @@ -35,12 +37,10 @@ extern "C" { #define INVALID_FD (-1) struct _signals_runtime_state { - volatile struct { - _Py_atomic_int tripped; - /* func is atomic to ensure that PyErr_SetInterrupt is async-signal-safe - * (even though it would probably be otherwise, anyway). - */ - _Py_atomic_address func; + struct { + // tripped and func should be accessed using atomic ops. + int tripped; + PyObject* func; } handlers[Py_NSIG]; volatile struct { @@ -60,8 +60,9 @@ struct _signals_runtime_state { #endif } wakeup; - /* Speed up sigcheck() when none tripped */ - _Py_atomic_int is_tripped; + /* Speed up sigcheck() when none tripped. + is_tripped should be accessed using atomic ops. */ + int is_tripped; /* These objects necessarily belong to the main interpreter. */ PyObject *default_handler; @@ -92,6 +93,15 @@ struct _signals_runtime_state { } +// Export for '_multiprocessing' shared extension +PyAPI_FUNC(int) _PyOS_IsMainThread(void); + +#ifdef MS_WINDOWS +// is not included by Python.h so use void* instead of HANDLE. +// Export for '_multiprocessing' shared extension +PyAPI_FUNC(void*) _PyOS_SigintEvent(void); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h index f427b4d695bd29..225f423912f2c2 100644 --- a/Include/internal/pycore_strhex.h +++ b/Include/internal/pycore_strhex.h @@ -9,21 +9,24 @@ extern "C" { #endif // Returns a str() containing the hex representation of argbuf. +// Export for '_hashlib' shared extension. PyAPI_FUNC(PyObject*) _Py_strhex(const char* argbuf, const Py_ssize_t arglen); // Returns a bytes() containing the ASCII hex representation of argbuf. -PyAPI_FUNC(PyObject*) _Py_strhex_bytes( +extern PyObject* _Py_strhex_bytes( const char* argbuf, const Py_ssize_t arglen); // These variants include support for a separator between every N bytes: -PyAPI_FUNC(PyObject*) _Py_strhex_with_sep( +extern PyObject* _Py_strhex_with_sep( const char* argbuf, const Py_ssize_t arglen, PyObject* sep, const int bytes_per_group); + +// Export for 'binascii' shared extension PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( const char* argbuf, const Py_ssize_t arglen, diff --git a/Include/internal/pycore_structseq.h b/Include/internal/pycore_structseq.h index 6f5dfc12707cf8..5cff165627502b 100644 --- a/Include/internal/pycore_structseq.h +++ b/Include/internal/pycore_structseq.h @@ -11,7 +11,8 @@ extern "C" { /* other API */ -PyAPI_FUNC(PyTypeObject *) _PyStructSequence_NewType( +// Export for '_curses' shared extension +PyAPI_FUNC(PyTypeObject*) _PyStructSequence_NewType( PyStructSequence_Desc *desc, unsigned long tp_flags); diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 9a005be5402c4e..1d782ca2c96e05 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -10,8 +10,17 @@ extern "C" { struct _mod; // Type defined in pycore_ast.h -typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, AnnotationBlock } - _Py_block_ty; +typedef enum _block_type { + FunctionBlock, ClassBlock, ModuleBlock, + // Used for annotations if 'from __future__ import annotations' is active. + // Annotation blocks cannot bind names and are not evaluated. + AnnotationBlock, + // Used for generics and type aliases. These work mostly like functions + // (see PEP 695 for details). The three different blocks function identically; + // they are different enum entries only so that error messages can be more + // precise. + TypeVarBoundBlock, TypeAliasBlock, TypeParamBlock +} _Py_block_ty; typedef enum _comprehension_type { NoComprehension = 0, @@ -49,7 +58,7 @@ typedef struct _symtable_entry { PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ - _Py_block_ty ste_type; /* module, class, function or annotation */ + _Py_block_ty ste_type; int ste_nested; /* true if block is nested */ unsigned ste_free : 1; /* true if block has free variables */ unsigned ste_child_free : 1; /* true if a child block has free vars, @@ -64,8 +73,12 @@ typedef struct _symtable_entry { unsigned ste_needs_class_closure : 1; /* for class scopes, true if a closure over __class__ should be created */ + unsigned ste_needs_classdict : 1; /* for class scopes, true if a closure + over the class dict should be created */ unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */ unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ + unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an + enclosing class scope */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ int ste_lineno; /* first line of block */ int ste_col_offset; /* offset of first line of block */ @@ -82,12 +95,13 @@ extern PyTypeObject PySTEntry_Type; extern long _PyST_GetSymbol(PySTEntryObject *, PyObject *); extern int _PyST_GetScope(PySTEntryObject *, PyObject *); +extern int _PyST_IsFunctionLike(PySTEntryObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, PyObject *filename, PyFutureFeatures *future); -PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *); +extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); extern void _PySymtable_Free(struct symtable *); @@ -105,14 +119,16 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define DEF_IMPORT 2<<6 /* assignment occurred via import */ #define DEF_ANNOT 2<<7 /* this name is annotated */ #define DEF_COMP_ITER 2<<8 /* this name is a comprehension iteration variable */ +#define DEF_TYPE_PARAM 2<<9 /* this name is a type parameter */ +#define DEF_COMP_CELL 2<<10 /* this name is a cell in an inlined comprehension */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) /* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol table. GLOBAL is returned from PyST_GetScope() for either of them. - It is stored in ste_symbols at bits 12-15. + It is stored in ste_symbols at bits 13-16. */ -#define SCOPE_OFFSET 11 +#define SCOPE_OFFSET 12 #define SCOPE_MASK (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL) #define LOCAL 1 diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index b4b1febafa4479..9b8eafd3d6cfd3 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -8,17 +8,23 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -PyAPI_FUNC(int) _PySys_Audit( +// Export for '_pickle' shared extension +PyAPI_FUNC(PyObject*) _PySys_GetAttr(PyThreadState *tstate, PyObject *name); + +// Export for '_pickle' shared extension +PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); + +extern int _PySys_Audit( PyThreadState *tstate, const char *event, const char *argFormat, ...); -/* We want minimal exposure of this function, so use extern rather than - PyAPI_FUNC() to not export the symbol. */ +// _PySys_ClearAuditHooks() must not be exported: use extern rather than +// PyAPI_FUNC(). We want minimal exposure of this function. extern void _PySys_ClearAuditHooks(PyThreadState *tstate); -PyAPI_FUNC(int) _PySys_SetAttr(PyObject *, PyObject *); +extern int _PySys_SetAttr(PyObject *, PyObject *); extern int _PySys_ClearAttrString(PyInterpreterState *interp, const char *name, int verbose); diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 949170c4493799..dabbd7b41556cd 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -1,3 +1,46 @@ +// The _PyTime_t API is written to use timestamp and timeout values stored in +// various formats and to read clocks. +// +// The _PyTime_t type is an integer to support directly common arithmetic +// operations like t1 + t2. +// +// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type +// is signed to support negative timestamps. The supported range is around +// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the +// supported date range is around [1677-09-21; 2262-04-11]. +// +// Formats: +// +// * seconds +// * seconds as a floating pointer number (C double) +// * milliseconds (10^-3 seconds) +// * microseconds (10^-6 seconds) +// * 100 nanoseconds (10^-7 seconds) +// * nanoseconds (10^-9 seconds) +// * timeval structure, 1 microsecond resolution (10^-6 seconds) +// * timespec structure, 1 nanosecond resolution (10^-9 seconds) +// +// Integer overflows are detected and raise OverflowError. Conversion to a +// resolution worse than 1 nanosecond is rounded correctly with the requested +// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling +// (towards +inf), half even and up (away from zero). +// +// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so +// the caller doesn't have to handle errors and doesn't need to hold the GIL. +// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on +// overflow. +// +// Clocks: +// +// * System clock +// * Monotonic clock +// * Performance counter +// +// Operations like (t * k / q) with integers are implemented in a way to reduce +// the risk of integer overflow. Such operation is used to convert a clock +// value expressed in ticks with a frequency to _PyTime_t, like +// QueryPerformanceCounter() with QueryPerformanceFrequency(). + #ifndef Py_INTERNAL_TIME_H #define Py_INTERNAL_TIME_H #ifdef __cplusplus @@ -9,17 +52,329 @@ extern "C" { #endif -struct _time_runtime_state { -#ifdef HAVE_TIMES - int ticks_per_second_initialized; - long ticks_per_second; -#else - int _not_used; +#ifdef __clang__ +struct timeval; +#endif + +// _PyTime_t: Python timestamp with subsecond precision. It can be used to +// store a duration, and so indirectly a date (related to another date, like +// UNIX epoch). +typedef int64_t _PyTime_t; +// _PyTime_MIN nanoseconds is around -292.3 years +#define _PyTime_MIN INT64_MIN +// _PyTime_MAX nanoseconds is around +292.3 years +#define _PyTime_MAX INT64_MAX +#define _SIZEOF_PYTIME_T 8 + +typedef enum { + // Round towards minus infinity (-inf). + // For example, used to read a clock. + _PyTime_ROUND_FLOOR=0, + + // Round towards infinity (+inf). + // For example, used for timeout to wait "at least" N seconds. + _PyTime_ROUND_CEILING=1, + + // Round to nearest with ties going to nearest even integer. + // For example, used to round from a Python float. + _PyTime_ROUND_HALF_EVEN=2, + + // Round away from zero + // For example, used for timeout. _PyTime_ROUND_CEILING rounds + // -1e-9 to 0 milliseconds which causes bpo-31786 issue. + // _PyTime_ROUND_UP rounds -1e-9 to -1 millisecond which keeps + // the timeout sign as expected. select.poll(timeout) must block + // for negative values. + _PyTime_ROUND_UP=3, + + // _PyTime_ROUND_TIMEOUT (an alias for _PyTime_ROUND_UP) should be + // used for timeouts. + _PyTime_ROUND_TIMEOUT = _PyTime_ROUND_UP +} _PyTime_round_t; + + +// Convert a time_t to a PyLong. +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyLong_FromTime_t(time_t sec); + +// Convert a PyLong to a time_t. +// Export for '_datetime' shared extension +PyAPI_FUNC(time_t) _PyLong_AsTime_t(PyObject *obj); + +// Convert a number of seconds, int or float, to time_t. +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyTime_ObjectToTime_t( + PyObject *obj, + time_t *sec, + _PyTime_round_t); + +// Convert a number of seconds, int or float, to a timeval structure. +// usec is in the range [0; 999999] and rounded towards zero. +// For example, -1.2 is converted to (-2, 800000). +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyTime_ObjectToTimeval( + PyObject *obj, + time_t *sec, + long *usec, + _PyTime_round_t); + +// Convert a number of seconds, int or float, to a timespec structure. +// nsec is in the range [0; 999999999] and rounded towards zero. +// For example, -1.2 is converted to (-2, 800000000). +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(int) _PyTime_ObjectToTimespec( + PyObject *obj, + time_t *sec, + long *nsec, + _PyTime_round_t); + + +// Create a timestamp from a number of seconds. +// Export for '_socket' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); + +// Create a timestamp from a number of seconds in double. +// Export for '_socket' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); + +// Macro to create a timestamp from a number of seconds, no integer overflow. +// Only use the macro for small values, prefer _PyTime_FromSeconds(). +#define _PYTIME_FROMSECONDS(seconds) \ + ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) + +// Create a timestamp from a number of nanoseconds. +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); + +// Create a timestamp from a number of microseconds. +// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +extern _PyTime_t _PyTime_FromMicrosecondsClamp(_PyTime_t us); + +// Create a timestamp from nanoseconds (Python int). +// Export for '_lsprof' shared extension. +PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, + PyObject *obj); + +// Convert a number of seconds (Python float or int) to a timestamp. +// Raise an exception and return -1 on error, return 0 on success. +// Export for '_socket' shared extension. +PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, + PyObject *obj, + _PyTime_round_t round); + +// Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. +// Raise an exception and return -1 on error, return 0 on success. +// Export for 'select' shared extension. +PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, + PyObject *obj, + _PyTime_round_t round); + +// Convert a timestamp to a number of seconds as a C double. +// Export for '_socket' shared extension. +PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); + +// Convert timestamp to a number of milliseconds (10^-3 seconds). +// Export for '_ssl' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, + _PyTime_round_t round); + +// Convert timestamp to a number of microseconds (10^-6 seconds). +// Export for '_queue' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, + _PyTime_round_t round); + +// Convert timestamp to a number of nanoseconds (10^-9 seconds). +extern _PyTime_t _PyTime_AsNanoseconds(_PyTime_t t); + +#ifdef MS_WINDOWS +// Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). +extern _PyTime_t _PyTime_As100Nanoseconds(_PyTime_t t, + _PyTime_round_t round); #endif -}; + +// Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int +// object. +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(PyObject*) _PyTime_AsNanosecondsObject(_PyTime_t t); + +#ifndef MS_WINDOWS +// Create a timestamp from a timeval structure. +// Raise an exception and return -1 on overflow, return 0 on success. +extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); +#endif + +// Convert a timestamp to a timeval structure (microsecond resolution). +// tv_usec is always positive. +// Raise an exception and return -1 if the conversion overflowed, +// return 0 on success. +// Export for 'select' shared extension. +PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, + struct timeval *tv, + _PyTime_round_t round); + +// Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. +// On overflow, clamp tv_sec to _PyTime_t min/max. +// Export for 'select' shared extension. +PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, + struct timeval *tv, + _PyTime_round_t round); + +// Convert a timestamp to a number of seconds (secs) and microseconds (us). +// us is always positive. This function is similar to _PyTime_AsTimeval() +// except that secs is always a time_t type, whereas the timeval structure +// uses a C long for tv_sec on Windows. +// Raise an exception and return -1 if the conversion overflowed, +// return 0 on success. +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( + _PyTime_t t, + time_t *secs, + int *us, + _PyTime_round_t round); + +#if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) +// Create a timestamp from a timespec structure. +// Raise an exception and return -1 on overflow, return 0 on success. +extern int _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts); + +// Convert a timestamp to a timespec structure (nanosecond resolution). +// tv_nsec is always positive. +// Raise an exception and return -1 on error, return 0 on success. +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); + +// Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. +// On overflow, clamp tv_sec to _PyTime_t min/max. +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); +#endif + + +// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +extern _PyTime_t _PyTime_Add(_PyTime_t t1, _PyTime_t t2); + +// Structure used by time.get_clock_info() +typedef struct { + const char *implementation; + int monotonic; + int adjustable; + double resolution; +} _Py_clock_info_t; + +// Get the current time from the system clock. +// +// If the internal clock fails, silently ignore the error and return 0. +// On integer overflow, silently ignore the overflow and clamp the clock to +// [_PyTime_MIN; _PyTime_MAX]. +// +// Use _PyTime_GetSystemClockWithInfo() to check for failure. +// Export for '_random' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); + +// Get the current time from the system clock. +// On success, set *t and *info (if not NULL), and return 0. +// On error, raise an exception and return -1. +extern int _PyTime_GetSystemClockWithInfo( + _PyTime_t *t, + _Py_clock_info_t *info); + +// Get the time of a monotonic clock, i.e. a clock that cannot go backwards. +// The clock is not affected by system clock updates. The reference point of +// the returned value is undefined, so that only the difference between the +// results of consecutive calls is valid. +// +// If the internal clock fails, silently ignore the error and return 0. +// On integer overflow, silently ignore the overflow and clamp the clock to +// [_PyTime_MIN; _PyTime_MAX]. +// +// Use _PyTime_GetMonotonicClockWithInfo() to check for failure. +// Export for '_random' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); + +// Get the time of a monotonic clock, i.e. a clock that cannot go backwards. +// The clock is not affected by system clock updates. The reference point of +// the returned value is undefined, so that only the difference between the +// results of consecutive calls is valid. +// +// Fill info (if set) with information of the function used to get the time. +// +// Return 0 on success, raise an exception and return -1 on error. +// Export for '_testsinglephase' shared extension. +PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( + _PyTime_t *t, + _Py_clock_info_t *info); + + +// Converts a timestamp to the Gregorian time, using the local time zone. +// Return 0 on success, raise an exception and return -1 on error. +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); + +// Converts a timestamp to the Gregorian time, assuming UTC. +// Return 0 on success, raise an exception and return -1 on error. +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); + +// Get the performance counter: clock with the highest available resolution to +// measure a short duration. +// +// If the internal clock fails, silently ignore the error and return 0. +// On integer overflow, silently ignore the overflow and clamp the clock to +// [_PyTime_MIN; _PyTime_MAX]. +// +// Use _PyTime_GetPerfCounterWithInfo() to check for failure. +// Export for '_lsprof' shared extension. +PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); + +// Get the performance counter: clock with the highest available resolution to +// measure a short duration. +// +// Fill info (if set) with information of the function used to get the time. +// +// Return 0 on success, raise an exception and return -1 on error. +extern int _PyTime_GetPerfCounterWithInfo( + _PyTime_t *t, + _Py_clock_info_t *info); + + +// Create a deadline. +// Pseudo code: _PyTime_GetMonotonicClock() + timeout. +// Export for '_ssl' shared extension. +PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); + +// Get remaining time from a deadline. +// Pseudo code: deadline - _PyTime_GetMonotonicClock(). +// Export for '_ssl' shared extension. +PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); + + +// --- _PyTimeFraction ------------------------------------------------------- + +typedef struct { + _PyTime_t numer; + _PyTime_t denom; +} _PyTimeFraction; + +// Set a fraction. +// Return 0 on success. +// Return -1 if the fraction is invalid. +extern int _PyTimeFraction_Set( + _PyTimeFraction *frac, + _PyTime_t numer, + _PyTime_t denom); + +// Compute ticks * frac.numer / frac.denom. +// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +extern _PyTime_t _PyTimeFraction_Mul( + _PyTime_t ticks, + const _PyTimeFraction *frac); + +// Compute a clock resolution: frac.numer / frac.denom / 1e9. +extern double _PyTimeFraction_Resolution( + const _PyTimeFraction *frac); #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_TIME_H */ +#endif // !Py_INTERNAL_TIME_H diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index b9df8766736adf..571cd6249f2812 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -1,4 +1,4 @@ -/* Auto-generated by Tools/build/generate_token.py */ +// Auto-generated by Tools/build/generate_token.py /* Token types */ #ifndef Py_INTERNAL_TOKEN_H @@ -69,16 +69,16 @@ extern "C" { #define COLONEQUAL 53 #define EXCLAMATION 54 #define OP 55 -#define AWAIT 56 -#define ASYNC 57 -#define TYPE_IGNORE 58 -#define TYPE_COMMENT 59 -#define SOFT_KEYWORD 60 -#define FSTRING_START 61 -#define FSTRING_MIDDLE 62 -#define FSTRING_END 63 +#define TYPE_IGNORE 56 +#define TYPE_COMMENT 57 +#define SOFT_KEYWORD 58 +#define FSTRING_START 59 +#define FSTRING_MIDDLE 60 +#define FSTRING_END 61 +#define COMMENT 62 +#define NL 63 #define ERRORTOKEN 64 -#define N_TOKENS 68 +#define N_TOKENS 66 #define NT_OFFSET 256 /* Special definitions for cooperation with parser */ @@ -94,7 +94,7 @@ extern "C" { (x) == FSTRING_MIDDLE) -// Symbols exported for test_peg_generator +// Export these 4 symbols for 'test_peg_generator' PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ PyAPI_FUNC(int) _PyToken_OneChar(int); PyAPI_FUNC(int) _PyToken_TwoChars(int, int); diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index c393b2c136f2de..10922bff98bd4b 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -8,6 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +// Export for '_ctypes' shared extension +PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, PyObject **); + +// Export for 'pyexact' shared extension +PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); + /* Write the Python traceback into the file 'fd'. For example: Traceback (most recent call first): @@ -25,7 +31,7 @@ extern "C" { This function is signal safe. */ -PyAPI_FUNC(void) _Py_DumpTraceback( +extern void _Py_DumpTraceback( int fd, PyThreadState *tstate); @@ -52,7 +58,7 @@ PyAPI_FUNC(void) _Py_DumpTraceback( This function is signal safe. */ -PyAPI_FUNC(const char*) _Py_DumpTracebackThreads( +extern const char* _Py_DumpTracebackThreads( int fd, PyInterpreterState *interp, PyThreadState *current_tstate); @@ -64,23 +70,23 @@ PyAPI_FUNC(const char*) _Py_DumpTracebackThreads( string which is not ready (PyUnicode_WCHAR_KIND). This function is signal safe. */ -PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text); +extern void _Py_DumpASCII(int fd, PyObject *text); /* Format an integer as decimal into the file descriptor fd. This function is signal safe. */ -PyAPI_FUNC(void) _Py_DumpDecimal( +extern void _Py_DumpDecimal( int fd, size_t value); /* Format an integer as hexadecimal with width digits into fd file descriptor. The function is signal safe. */ -PyAPI_FUNC(void) _Py_DumpHexadecimal( +extern void _Py_DumpHexadecimal( int fd, uintptr_t value, Py_ssize_t width); -PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame( +extern PyObject* _PyTraceBack_FromFrame( PyObject *tb_next, PyFrameObject *frame); @@ -89,11 +95,10 @@ PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame( /* Write the traceback tb to file f. Prefix each line with indent spaces followed by the margin (if it is not NULL). */ -PyAPI_FUNC(int) _PyTraceBack_Print_Indented( - PyObject *tb, int indent, const char* margin, - const char *header_margin, const char *header, PyObject *f); -PyAPI_FUNC(int) _Py_WriteIndentedMargin(int, const char*, PyObject *); -PyAPI_FUNC(int) _Py_WriteIndent(int, PyObject *); +extern int _PyTraceBack_Print( + PyObject *tb, const char *header, PyObject *f); +extern int _Py_WriteIndentedMargin(int, const char*, PyObject *); +extern int _Py_WriteIndent(int, PyObject *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index d086adc61c319b..7ddc5bac5d10af 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -117,6 +117,53 @@ struct _tracemalloc_runtime_state { } +// Get the traceback where a memory block was allocated. +// +// Return a tuple of (filename: str, lineno: int) tuples. +// +// Return None if the tracemalloc module is disabled or if the memory block +// is not tracked by tracemalloc. +// +// Raise an exception and return NULL on error. +// +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( + unsigned int domain, + uintptr_t ptr); + +/* Return non-zero if tracemalloc is tracing */ +extern int _PyTraceMalloc_IsTracing(void); + +/* Clear the tracemalloc traces */ +extern void _PyTraceMalloc_ClearTraces(void); + +/* Clear the tracemalloc traces */ +extern PyObject* _PyTraceMalloc_GetTraces(void); + +/* Clear tracemalloc traceback for an object */ +extern PyObject* _PyTraceMalloc_GetObjectTraceback(PyObject *obj); + +/* Initialize tracemalloc */ +extern int _PyTraceMalloc_Init(void); + +/* Start tracemalloc */ +extern int _PyTraceMalloc_Start(int max_nframe); + +/* Stop tracemalloc */ +extern void _PyTraceMalloc_Stop(void); + +/* Get the tracemalloc traceback limit */ +extern int _PyTraceMalloc_GetTracebackLimit(void); + +/* Get the memory usage of tracemalloc in bytes */ +extern size_t _PyTraceMalloc_GetMemory(void); + +/* Get the current size and peak size of traced memory blocks as a 2-tuple */ +extern PyObject* _PyTraceMalloc_GetTracedMemory(void); + +/* Set the peak size of traced memory blocks to the current size */ +extern void _PyTraceMalloc_ResetPeak(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h new file mode 100644 index 00000000000000..17f3e865641773 --- /dev/null +++ b/Include/internal/pycore_tstate.h @@ -0,0 +1,26 @@ +#ifndef Py_INTERNAL_TSTATE_H +#define Py_INTERNAL_TSTATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The +// PyThreadState fields are exposed as part of the C API, although most fields +// are intended to be private. The _PyThreadStateImpl fields not exposed. +typedef struct _PyThreadStateImpl { + // semi-public fields are in PyThreadState. + PyThreadState base; + + // TODO: add private fields here +} _PyThreadStateImpl; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_TSTATE_H */ diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 335edad89792c3..4fa7a12206bcb2 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -8,8 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "tupleobject.h" /* _PyTuple_CAST() */ - +extern void _PyTuple_MaybeUntrack(PyObject *); +extern void _PyTuple_DebugMallocStats(FILE *out); /* runtime lifecycle */ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c7d00002fbc63c..f983de56049631 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -4,12 +4,12 @@ extern "C" { #endif -#include "pycore_moduleobject.h" - #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_moduleobject.h" // PyModuleObject + /* state */ @@ -46,11 +46,9 @@ typedef struct { PyTypeObject *type; int readying; int ready; - // XXX tp_dict, tp_bases, and tp_mro can probably be statically - // allocated, instead of dynamically and stored on the interpreter. + // XXX tp_dict can probably be statically allocated, + // instead of dynamically and stored on the interpreter. PyObject *tp_dict; - PyObject *tp_bases; - PyObject *tp_mro; PyObject *tp_subclasses; /* We never clean up weakrefs for static builtin types since they will effectively never get triggered. However, there @@ -116,7 +114,10 @@ extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTyp extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); +// Export for 'math' shared extension, used via _PyType_IsReady() static inline +// function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); + extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); @@ -130,18 +131,24 @@ _PyType_IsReady(PyTypeObject *type) return _PyType_GetDict(type) != NULL; } -PyObject * -_Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); -PyObject * -_Py_type_getattro(PyTypeObject *type, PyObject *name); +extern PyObject* _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, + int *suppress_missing_attribute); +extern PyObject* _Py_type_getattro(PyTypeObject *type, PyObject *name); + +extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op); + +extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); +extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); + +extern PyTypeObject _PyBufferWrapper_Type; + +extern PyObject* _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, + PyObject *name, int *meth_found); -PyObject *_Py_slot_tp_getattro(PyObject *self, PyObject *name); -PyObject *_Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); -PyAPI_DATA(PyTypeObject) _PyBufferWrapper_Type; +// This is exported for the _testinternalcapi module. +PyAPI_FUNC(PyObject *) _PyType_GetModuleName(PyTypeObject *); -PyObject * -_PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); #ifdef __cplusplus } diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h new file mode 100644 index 00000000000000..c9fa97d6820757 --- /dev/null +++ b/Include/internal/pycore_typevarobject.h @@ -0,0 +1,24 @@ +#ifndef Py_INTERNAL_TYPEVAROBJECT_H +#define Py_INTERNAL_TYPEVAROBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern PyObject *_Py_make_typevar(PyObject *, PyObject *, PyObject *); +extern PyObject *_Py_make_paramspec(PyThreadState *, PyObject *); +extern PyObject *_Py_make_typevartuple(PyThreadState *, PyObject *); +extern PyObject *_Py_make_typealias(PyThreadState *, PyObject *); +extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); +extern int _Py_initialize_generic(PyInterpreterState *); +extern void _Py_clear_generic_types(PyInterpreterState *); + +extern PyTypeObject _PyTypeAlias_Type; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_TYPEVAROBJECT_H */ diff --git a/Include/internal/pycore_ucnhash.h b/Include/internal/pycore_ucnhash.h index 187dd68e7347ff..1561dfbb3150d3 100644 --- a/Include/internal/pycore_ucnhash.h +++ b/Include/internal/pycore_ucnhash.h @@ -28,6 +28,8 @@ typedef struct { } _PyUnicode_Name_CAPI; +extern _PyUnicode_Name_CAPI* _PyUnicode_GetNameCAPI(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 1bb0f366e78163..7ee540154b23d8 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -8,13 +8,264 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex #include "pycore_fileutils.h" // _Py_error_handler +#include "pycore_identifier.h" // _Py_Identifier #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI -void _PyUnicode_ExactDealloc(PyObject *op); -Py_ssize_t _PyUnicode_InternedSize(void); +/* --- Characters Type APIs ----------------------------------------------- */ -/* runtime lifecycle */ +extern int _PyUnicode_IsXidStart(Py_UCS4 ch); +extern int _PyUnicode_IsXidContinue(Py_UCS4 ch); +extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch); +extern int _PyUnicode_IsCased(Py_UCS4 ch); + +/* --- Unicode API -------------------------------------------------------- */ + +// Export for '_json' shared extension +PyAPI_FUNC(int) _PyUnicode_CheckConsistency( + PyObject *op, + int check_content); + +extern void _PyUnicode_ExactDealloc(PyObject *op); +extern Py_ssize_t _PyUnicode_InternedSize(void); + +// Get a copy of a Unicode string. +// Export for '_datetime' shared extension. +PyAPI_FUNC(PyObject*) _PyUnicode_Copy( + PyObject *unicode); + +/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash + if parameters are invalid (e.g. if length is longer than the string). */ +extern void _PyUnicode_FastFill( + PyObject *unicode, + Py_ssize_t start, + Py_ssize_t length, + Py_UCS4 fill_char + ); + +/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so + may crash if parameters are invalid (e.g. if the output string + is too short). */ +extern void _PyUnicode_FastCopyCharacters( + PyObject *to, + Py_ssize_t to_start, + PyObject *from, + Py_ssize_t from_start, + Py_ssize_t how_many + ); + +/* Create a new string from a buffer of ASCII characters. + WARNING: Don't check if the string contains any non-ASCII character. */ +extern PyObject* _PyUnicode_FromASCII( + const char *buffer, + Py_ssize_t size); + +/* Compute the maximum character of the substring unicode[start:end]. + Return 127 for an empty string. */ +extern Py_UCS4 _PyUnicode_FindMaxChar ( + PyObject *unicode, + Py_ssize_t start, + Py_ssize_t end); + +/* --- _PyUnicodeWriter API ----------------------------------------------- */ + +/* Format the object based on the format_spec, as defined in PEP 3101 + (Advanced String Formatting). */ +extern int _PyUnicode_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); + +/* --- UTF-7 Codecs ------------------------------------------------------- */ + +extern PyObject* _PyUnicode_EncodeUTF7( + PyObject *unicode, /* Unicode object */ + int base64SetO, /* Encode RFC2152 Set O characters in base64 */ + int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ + const char *errors); /* error handling */ + +/* --- UTF-8 Codecs ------------------------------------------------------- */ + +// Export for '_tkinter' shared extension. +PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String( + PyObject *unicode, + const char *errors); + +/* --- UTF-32 Codecs ------------------------------------------------------ */ + +// Export for '_tkinter' shared extension +PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32( + PyObject *object, /* Unicode object */ + const char *errors, /* error handling */ + int byteorder); /* byteorder to use 0=BOM+native;-1=LE,1=BE */ + +/* --- UTF-16 Codecs ------------------------------------------------------ */ + +// Returns a Python string object holding the UTF-16 encoded value of +// the Unicode data. +// +// If byteorder is not 0, output is written according to the following +// byte order: +// +// byteorder == -1: little endian +// byteorder == 0: native byte order (writes a BOM mark) +// byteorder == 1: big endian +// +// If byteorder is 0, the output string will always start with the +// Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is +// prepended. +// +// Export for '_tkinter' shared extension +PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( + PyObject* unicode, /* Unicode object */ + const char *errors, /* error handling */ + int byteorder); /* byteorder to use 0=BOM+native;-1=LE,1=BE */ + +/* --- Unicode-Escape Codecs ---------------------------------------------- */ + +/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */ +extern PyObject* _PyUnicode_DecodeUnicodeEscapeStateful( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed); /* bytes consumed */ + +// Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape +// chars. +// Export for test_peg_generator. +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed, /* bytes consumed */ + const char **first_invalid_escape); /* on return, points to first + invalid escaped char in + string. */ + +/* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */ + +/* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */ +extern PyObject* _PyUnicode_DecodeRawUnicodeEscapeStateful( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed); /* bytes consumed */ + +/* --- Latin-1 Codecs ----------------------------------------------------- */ + +extern PyObject* _PyUnicode_AsLatin1String( + PyObject* unicode, + const char* errors); + +/* --- ASCII Codecs ------------------------------------------------------- */ + +extern PyObject* _PyUnicode_AsASCIIString( + PyObject* unicode, + const char* errors); + +/* --- Character Map Codecs ----------------------------------------------- */ + +/* Translate an Unicode object by applying a character mapping table to + it and return the resulting Unicode object. + + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). + + Mapping tables may be dictionaries or sequences. Unmapped character + ordinals (ones which cause a LookupError) are left untouched and + are copied as-is. +*/ +extern PyObject* _PyUnicode_EncodeCharmap( + PyObject *unicode, /* Unicode object */ + PyObject *mapping, /* encoding mapping */ + const char *errors); /* error handling */ + +/* --- Decimal Encoder ---------------------------------------------------- */ + +// Coverts a Unicode object holding a decimal value to an ASCII string +// for using in int, float and complex parsers. +// Transforms code points that have decimal digit property to the +// corresponding ASCII digit code points. Transforms spaces to ASCII. +// Transforms code points starting from the first non-ASCII code point that +// is neither a decimal digit nor a space to the end into '?'. +// +// Export for '_testinternalcapi' shared extension. +PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( + PyObject *unicode); /* Unicode object */ + +/* --- Methods & Slots ---------------------------------------------------- */ + +extern PyObject* _PyUnicode_JoinArray( + PyObject *separator, + PyObject *const *items, + Py_ssize_t seqlen + ); + +/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, + 0 otherwise. The right argument must be ASCII identifier. + Any error occurs inside will be cleared before return. */ +extern int _PyUnicode_EqualToASCIIId( + PyObject *left, /* Left string */ + _Py_Identifier *right /* Right identifier */ + ); + +// Test whether a unicode is equal to ASCII string. Return 1 if true, +// 0 otherwise. The right argument must be ASCII-encoded string. +// Any error occurs inside will be cleared before return. +// Export for '_ctypes' shared extension +PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString( + PyObject *left, + const char *right /* ASCII-encoded string */ + ); + +/* Externally visible for str.strip(unicode) */ +extern PyObject* _PyUnicode_XStrip( + PyObject *self, + int striptype, + PyObject *sepobj + ); + + +/* Using explicit passed-in values, insert the thousands grouping + into the string pointed to by buffer. For the argument descriptions, + see Objects/stringlib/localeutil.h */ +extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( + _PyUnicodeWriter *writer, + Py_ssize_t n_buffer, + PyObject *digits, + Py_ssize_t d_pos, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + PyObject *thousands_sep, + Py_UCS4 *maxchar); + +/* --- Misc functions ----------------------------------------------------- */ + +extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int); + +/* Fast equality check when the inputs are known to be exact unicode types + and where the hash values are equal (i.e. a very probable match) */ +extern int _PyUnicode_EQ(PyObject *, PyObject *); + +// Equality check. +// Export for '_pickle' shared extension. +PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *); + +extern int _PyUnicode_WideCharString_Converter(PyObject *, void *); +extern int _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *); + +// Export for test_peg_generator +PyAPI_FUNC(Py_ssize_t) _PyUnicode_ScanIdentifier(PyObject *); + +/* --- Runtime lifecycle -------------------------------------------------- */ extern void _PyUnicode_InitState(PyInterpreterState *); extern PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *); @@ -24,10 +275,10 @@ extern void _PyUnicode_FiniTypes(PyInterpreterState *); extern PyTypeObject _PyUnicodeASCIIIter_Type; -/* other API */ +/* --- Other API ---------------------------------------------------------- */ struct _Py_unicode_runtime_ids { - PyThread_type_lock lock; + PyMutex mutex; // next_index value must be preserved when Py_Initialize()/Py_Finalize() // is called multiple times: see _PyUnicode_FromId() implementation. Py_ssize_t next_index; @@ -63,6 +314,10 @@ struct _Py_unicode_state { extern void _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p); extern void _PyUnicode_ClearInterned(PyInterpreterState *interp); +// Like PyUnicode_AsUTF8(), but check for embedded null characters. +// Export for '_sqlite3' shared extension. +PyAPI_FUNC(const char *) _PyUnicode_AsUTF8NoNUL(PyObject *); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 8e086edbdf8193..be9baa3eebecfc 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -123,6 +123,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__classcell__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__classdict__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__classdictcell__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__complex__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -309,6 +315,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__main__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__match_args__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__matmul__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -489,6 +498,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__trunc__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__type_params__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__typing_is_unpacked_typevartuple__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -639,6 +651,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(access); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(aclose); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(add); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -669,9 +684,15 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(as_integer_ratio); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(asend); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(ast); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(athrow); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(attribute); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -702,6 +723,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(block); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(bound); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(buffer); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -762,6 +786,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(call_soon); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(callback); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(cancel); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -867,6 +894,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(context); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(contravariant); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(cookie); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -882,6 +912,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(count); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(covariant); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(cwd); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -966,9 +999,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(dst_dir_fd); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(duration); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(e); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1014,24 +1044,27 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(eventmask); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(exc_type); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(exc_value); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(excepthook); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(exception); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(existing_file_name); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(exp); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(extend); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(extra_tokens); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(f); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(facility); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1101,9 +1134,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(format); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(frequency); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(from_param); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1125,6 +1155,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(future); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(g); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(generation); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1164,6 +1197,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(groups); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(h); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(handle); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1188,6 +1224,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(ident); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(identity_hint); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(ignore); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1209,6 +1248,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(inf); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(infer_variance); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(inheritable); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1245,6 +1287,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(intersection); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(interval); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(is_running); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1476,6 +1521,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(ndigits); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(new_file_name); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(new_limit); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1488,6 +1536,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(next); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(nlocals); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(node_depth); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1557,6 +1608,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(order); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(origin); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(out_fd); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1632,6 +1686,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(progress_handler); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(progress_routine); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(proto); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1809,9 +1866,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(sort); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(sound); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(source); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1848,6 +1902,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(step); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(steps); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(store_name); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1932,9 +1989,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(trace_callback); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(traceback); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(trailers); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1956,6 +2010,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(type); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(type_params); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(tz); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h new file mode 100644 index 00000000000000..ea8f90bf8c1d8f --- /dev/null +++ b/Include/internal/pycore_uops.h @@ -0,0 +1,35 @@ +#ifndef Py_INTERNAL_UOPS_H +#define Py_INTERNAL_UOPS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_frame.h" // _PyInterpreterFrame + +#define _Py_UOP_MAX_TRACE_LENGTH 512 + +typedef struct { + uint16_t opcode; + uint16_t oparg; + uint32_t target; + uint64_t operand; // A cache entry +} _PyUOpInstruction; + +typedef struct { + _PyExecutorObject base; + _PyUOpInstruction trace[1]; +} _PyUOpExecutorObject; + +_PyInterpreterFrame *_PyUOpExecute( + _PyExecutorObject *executor, + _PyInterpreterFrame *frame, + PyObject **stack_pointer); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UOPS_H */ diff --git a/Include/internal/pycore_warnings.h b/Include/internal/pycore_warnings.h index efb4f1cd7eac80..9785d7cc467de2 100644 --- a/Include/internal/pycore_warnings.h +++ b/Include/internal/pycore_warnings.h @@ -19,9 +19,10 @@ struct _warnings_runtime_state { extern int _PyWarnings_InitState(PyInterpreterState *interp); -PyAPI_FUNC(PyObject*) _PyWarnings_Init(void); +extern PyObject* _PyWarnings_Init(void); extern void _PyErr_WarnUnawaitedCoroutine(PyObject *coro); +extern void _PyErr_WarnUnawaitedAgenMethod(PyAsyncGenObject *agen, PyObject *method); #ifdef __cplusplus } diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h new file mode 100644 index 00000000000000..eacbe14c903289 --- /dev/null +++ b/Include/internal/pycore_weakref.h @@ -0,0 +1,64 @@ +#ifndef Py_INTERNAL_WEAKREF_H +#define Py_INTERNAL_WEAKREF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() + +static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { + assert(PyWeakref_Check(ref_obj)); + PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); + PyObject *obj = ref->wr_object; + + if (obj == Py_None) { + // clear_weakref() was called + return NULL; + } + + // Explanation for the Py_REFCNT() check: when a weakref's target is part + // of a long chain of deallocations which triggers the trashcan mechanism, + // clearing the weakrefs can be delayed long after the target's refcount + // has dropped to zero. In the meantime, code accessing the weakref will + // be able to "see" the target object even though it is supposed to be + // unreachable. See issue gh-60806. + Py_ssize_t refcnt = Py_REFCNT(obj); + if (refcnt == 0) { + return NULL; + } + + assert(refcnt > 0); + return Py_NewRef(obj); +} + +static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { + assert(PyWeakref_Check(ref_obj)); + int is_dead; + Py_BEGIN_CRITICAL_SECTION(ref_obj); + PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); + PyObject *obj = ref->wr_object; + if (obj == Py_None) { + // clear_weakref() was called + is_dead = 1; + } + else { + // See _PyWeakref_GET_REF() for the rationale of this test + is_dead = (Py_REFCNT(obj) == 0); + } + Py_END_CRITICAL_SECTION(); + return is_dead; +} + +extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyWeakReference *head); + +extern void _PyWeakref_ClearRef(PyWeakReference *self); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_WEAKREF_H */ + diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h new file mode 100644 index 00000000000000..8432632f339e92 --- /dev/null +++ b/Include/interpreteridobject.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERPRETERIDOBJECT_H +#define Py_INTERPRETERIDOBJECT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_INTERPRETERIDOBJECT_H +# include "cpython/interpreteridobject.h" +# undef Py_CPYTHON_INTERPRETERIDOBJECT_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Include/intrcheck.h b/Include/intrcheck.h index b8cc65601683cb..1d1feee83de483 100644 --- a/Include/intrcheck.h +++ b/Include/intrcheck.h @@ -5,6 +5,7 @@ extern "C" { #endif PyAPI_FUNC(int) PyOS_InterruptOccurred(void); + #ifdef HAVE_FORK #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 PyAPI_FUNC(void) PyOS_BeforeFork(void); @@ -12,18 +13,10 @@ PyAPI_FUNC(void) PyOS_AfterFork_Parent(void); PyAPI_FUNC(void) PyOS_AfterFork_Child(void); #endif #endif + /* Deprecated, please use PyOS_AfterFork_Child() instead */ Py_DEPRECATED(3.7) PyAPI_FUNC(void) PyOS_AfterFork(void); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyOS_IsMainThread(void); - -#ifdef MS_WINDOWS -/* windows.h is not included by Python.h so use void* instead of HANDLE */ -PyAPI_FUNC(void*) _PyOS_SigintEvent(void); -#endif -#endif /* !Py_LIMITED_API */ - #ifdef __cplusplus } #endif diff --git a/Include/iterobject.h b/Include/iterobject.h index fff30f7176fdeb..e69d09719bb4d1 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -7,9 +7,6 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; -#ifdef Py_BUILD_CORE -extern PyTypeObject _PyAnextAwaitable_Type; -#endif #define PySeqIter_Check(op) Py_IS_TYPE((op), &PySeqIter_Type) diff --git a/Include/longobject.h b/Include/longobject.h index e559e238ae5a35..51005efff636fa 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -7,7 +7,7 @@ extern "C" { /* Long (arbitrary precision) integer object interface */ -PyAPI_DATA(PyTypeObject) PyLong_Type; +// PyLong_Type is declared by object.h #define PyLong_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS) @@ -18,12 +18,18 @@ PyAPI_FUNC(PyObject *) PyLong_FromUnsignedLong(unsigned long); PyAPI_FUNC(PyObject *) PyLong_FromSize_t(size_t); PyAPI_FUNC(PyObject *) PyLong_FromSsize_t(Py_ssize_t); PyAPI_FUNC(PyObject *) PyLong_FromDouble(double); + PyAPI_FUNC(long) PyLong_AsLong(PyObject *); PyAPI_FUNC(long) PyLong_AsLongAndOverflow(PyObject *, int *); PyAPI_FUNC(Py_ssize_t) PyLong_AsSsize_t(PyObject *); PyAPI_FUNC(size_t) PyLong_AsSize_t(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLong(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLongMask(PyObject *); + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PyLong_AsInt(PyObject *); +#endif + PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); /* It may be useful in the future. I've added it in the PyInt -> PyLong diff --git a/Include/methodobject.h b/Include/methodobject.h index 72af5ad933df7f..2381e8482b82a8 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -49,8 +49,6 @@ PyAPI_FUNC(PyCFunction) PyCFunction_GetFunction(PyObject *); PyAPI_FUNC(PyObject *) PyCFunction_GetSelf(PyObject *); PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *); -Py_DEPRECATED(3.9) PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *); - struct PyMethodDef { const char *ml_name; /* The name of the built-in function/method */ PyCFunction ml_meth; /* The C function that implements it */ diff --git a/Include/modsupport.h b/Include/modsupport.h index 4e369bd56b4d20..ea4c0fce9f4562 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -1,3 +1,4 @@ +// Module support interface #ifndef Py_MODSUPPORT_H #define Py_MODSUPPORT_H @@ -5,47 +6,34 @@ extern "C" { #endif -/* Module support interface */ - -#include // va_list - -/* If PY_SSIZE_T_CLEAN is defined, each functions treats #-specifier - to mean Py_ssize_t */ -#ifdef PY_SSIZE_T_CLEAN -#define PyArg_Parse _PyArg_Parse_SizeT -#define PyArg_ParseTuple _PyArg_ParseTuple_SizeT -#define PyArg_ParseTupleAndKeywords _PyArg_ParseTupleAndKeywords_SizeT -#define PyArg_VaParse _PyArg_VaParse_SizeT -#define PyArg_VaParseTupleAndKeywords _PyArg_VaParseTupleAndKeywords_SizeT -#define Py_BuildValue _Py_BuildValue_SizeT -#define Py_VaBuildValue _Py_VaBuildValue_SizeT -#endif - -/* Due to a glitch in 3.2, the _SizeT versions weren't exported from the DLL. */ -#if !defined(PY_SSIZE_T_CLEAN) || !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...); PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...); PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, - const char *, char **, ...); + const char *, PY_CXX_CONST char * const *, ...); PyAPI_FUNC(int) PyArg_VaParse(PyObject *, const char *, va_list); PyAPI_FUNC(int) PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, - const char *, char **, va_list); -#endif + const char *, PY_CXX_CONST char * const *, va_list); + PyAPI_FUNC(int) PyArg_ValidateKeywordArguments(PyObject *); PyAPI_FUNC(int) PyArg_UnpackTuple(PyObject *, const char *, Py_ssize_t, Py_ssize_t, ...); PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...); -PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); - - PyAPI_FUNC(PyObject *) Py_VaBuildValue(const char *, va_list); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030a0000 // Add an attribute with name 'name' and value 'obj' to the module 'mod. -// On success, return 0 on success. +// On success, return 0. // On error, raise an exception and return -1. PyAPI_FUNC(int) PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value); +#endif /* Py_LIMITED_API */ + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +// Similar to PyModule_AddObjectRef() but steal a reference to 'value'. +PyAPI_FUNC(int) PyModule_Add(PyObject *mod, const char *name, PyObject *value); +#endif /* Py_LIMITED_API */ -// Similar to PyModule_AddObjectRef() but steal a reference to 'obj' -// (Py_DECREF(obj)) on success (if it returns 0). +// Similar to PyModule_AddObjectRef() and PyModule_Add() but steal +// a reference to 'value' on success and only on success. +// Errorprone. Should not be used in new code. PyAPI_FUNC(int) PyModule_AddObject(PyObject *mod, const char *, PyObject *value); PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long); @@ -120,14 +108,6 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); #define PYTHON_ABI_VERSION 3 #define PYTHON_ABI_STRING "3" -#ifdef Py_TRACE_REFS - /* When we are tracing reference counts, rename module creation functions so - modules compiled with incompatible settings will generate a - link-time error. */ - #define PyModule_Create2 PyModule_Create2TraceRefs - #define PyModule_FromDefAndSpec2 PyModule_FromDefAndSpec2TraceRefs -#endif - PyAPI_FUNC(PyObject *) PyModule_Create2(PyModuleDef*, int apiver); #ifdef Py_LIMITED_API @@ -154,12 +134,6 @@ PyAPI_FUNC(PyObject *) PyModule_FromDefAndSpec2(PyModuleDef *def, #endif /* New in 3.5 */ -#ifndef Py_LIMITED_API -# define Py_CPYTHON_MODSUPPORT_H -# include "cpython/modsupport.h" -# undef Py_CPYTHON_MODSUPPORT_H -#endif - #ifdef __cplusplus } #endif diff --git a/Include/moduleobject.h b/Include/moduleobject.h index b8bdfe29d80406..42b87cc4e91012 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -27,11 +27,6 @@ PyAPI_FUNC(PyObject *) PyModule_GetNameObject(PyObject *); PyAPI_FUNC(const char *) PyModule_GetName(PyObject *); Py_DEPRECATED(3.2) PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *); PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyModule_Clear(PyObject *); -PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *); -PyAPI_FUNC(int) _PyModuleSpec_IsInitializing(PyObject *); -#endif PyAPI_FUNC(PyModuleDef*) PyModule_GetDef(PyObject*); PyAPI_FUNC(void*) PyModule_GetState(PyObject*); @@ -78,19 +73,23 @@ struct PyModuleDef_Slot { #define Py_mod_create 1 #define Py_mod_exec 2 -#define Py_mod_multiple_interpreters 3 +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 +# define Py_mod_multiple_interpreters 3 +#endif #ifndef Py_LIMITED_API #define _Py_mod_LAST_SLOT 3 #endif -/* for Py_mod_multiple_interpreters: */ -#define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) -#define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) -#define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) - #endif /* New in 3.5 */ +/* for Py_mod_multiple_interpreters: */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 +# define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) +# define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) +# define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) +#endif + struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; @@ -103,12 +102,6 @@ struct PyModuleDef { freefunc m_free; }; - -// Internal C API -#ifdef Py_BUILD_CORE -extern int _PyModule_IsExtension(PyObject *obj); -#endif - #ifdef __cplusplus } #endif diff --git a/Include/object.h b/Include/object.h index 81aeb2d8bd5a69..bd576b0bd43211 100644 --- a/Include/object.h +++ b/Include/object.h @@ -51,30 +51,11 @@ A standard interface exists for objects that contain an array of items whose size is determined when the object is allocated. */ -#include "pystats.h" - /* Py_DEBUG implies Py_REF_DEBUG. */ #if defined(Py_DEBUG) && !defined(Py_REF_DEBUG) # define Py_REF_DEBUG #endif -#if defined(Py_LIMITED_API) && defined(Py_TRACE_REFS) -# error Py_LIMITED_API is incompatible with Py_TRACE_REFS -#endif - -#ifdef Py_TRACE_REFS -/* Define pointers to support a doubly-linked list of all live heap objects. */ -#define _PyObject_HEAD_EXTRA \ - PyObject *_ob_next; \ - PyObject *_ob_prev; - -#define _PyObject_EXTRA_INIT _Py_NULL, _Py_NULL, - -#else -# define _PyObject_HEAD_EXTRA -# define _PyObject_EXTRA_INIT -#endif - /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; @@ -107,7 +88,7 @@ having all the lower 32 bits set, which will avoid the reference count to go beyond the refcount limit. Immortality checks for reference count decreases will be done by checking the bit sign flag in the lower 32 bits. */ -#define _Py_IMMORTAL_REFCNT UINT_MAX +#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX) #else /* @@ -122,22 +103,40 @@ immortality, but the execution would still be correct. Reference count increases and decreases will first go through an immortality check by comparing the reference count field to the immortality reference count. */ -#define _Py_IMMORTAL_REFCNT (UINT_MAX >> 2) +#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX >> 2) +#endif + +// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is +// always 32-bits. +#ifdef Py_GIL_DISABLED +#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX #endif +// Kept for backward compatibility. It was needed by Py_TRACE_REFS build. +#define _PyObject_EXTRA_INIT + // Make all internal uses of PyObject_HEAD_INIT immortal while preserving the // C-API expectation that the refcnt will be set to 1. -#ifdef Py_BUILD_CORE +#if defined(Py_GIL_DISABLED) +#define PyObject_HEAD_INIT(type) \ + { \ + 0, \ + 0, \ + { 0 }, \ + 0, \ + _Py_IMMORTAL_REFCNT_LOCAL, \ + 0, \ + (type), \ + }, +#elif defined(Py_BUILD_CORE) #define PyObject_HEAD_INIT(type) \ { \ - _PyObject_EXTRA_INIT \ { _Py_IMMORTAL_REFCNT }, \ (type) \ }, #else #define PyObject_HEAD_INIT(type) \ { \ - _PyObject_EXTRA_INIT \ { 1 }, \ (type) \ }, @@ -163,16 +162,65 @@ check by comparing the reference count field to the immortality reference count. * by hand. Similarly every pointer to a variable-size Python object can, * in addition, be cast to PyVarObject*. */ +#ifndef Py_GIL_DISABLED struct _object { - _PyObject_HEAD_EXTRA +#if (defined(__GNUC__) || defined(__clang__)) \ + && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L) + // On C99 and older, anonymous union is a GCC and clang extension + __extension__ +#endif +#ifdef _MSC_VER + // Ignore MSC warning C4201: "nonstandard extension used: + // nameless struct/union" + __pragma(warning(push)) + __pragma(warning(disable: 4201)) +#endif union { Py_ssize_t ob_refcnt; #if SIZEOF_VOID_P > 4 PY_UINT32_T ob_refcnt_split[2]; #endif }; +#ifdef _MSC_VER + __pragma(warning(pop)) +#endif + PyTypeObject *ob_type; }; +#else +// Objects that are not owned by any thread use a thread id (tid) of zero. +// This includes both immortal objects and objects whose reference count +// fields have been merged. +#define _Py_UNOWNED_TID 0 + +// The shared reference count uses the two least-significant bits to store +// flags. The remaining bits are used to store the reference count. +#define _Py_REF_SHARED_SHIFT 2 +#define _Py_REF_SHARED_FLAG_MASK 0x3 + +// The shared flags are initialized to zero. +#define _Py_REF_SHARED_INIT 0x0 +#define _Py_REF_MAYBE_WEAKREF 0x1 +#define _Py_REF_QUEUED 0x2 +#define _Py_REF_MERGED 0x3 + +// Create a shared field from a refcnt and desired flags +#define _Py_REF_SHARED(refcnt, flags) (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) + +// NOTE: In non-free-threaded builds, `struct _PyMutex` is defined in +// pycore_lock.h. See pycore_lock.h for more details. +struct _PyMutex { uint8_t v; }; + +struct _object { + uintptr_t ob_tid; // thread id (or zero) + uint16_t _padding; + struct _PyMutex ob_mutex; // per-object lock + uint8_t ob_gc_bits; // gc-related state + uint32_t ob_ref_local; // local reference count + Py_ssize_t ob_ref_shared; // shared (atomic) reference count + PyTypeObject *ob_type; +}; +#endif /* Cast argument to PyObject* type. */ #define _PyObject_CAST(op) _Py_CAST(PyObject*, (op)) @@ -190,9 +238,76 @@ typedef struct { PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); #define Py_Is(x, y) ((x) == (y)) +#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +static inline uintptr_t +_Py_ThreadId(void) +{ + uintptr_t tid; +#if defined(_MSC_VER) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(_MSC_VER) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(_MSC_VER) && defined(_M_ARM64) + tid = __getReg(18); +#elif defined(__i386__) + __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS +#elif defined(__MACH__) && defined(__x86_64__) + __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS +#elif defined(__x86_64__) + __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS +#elif defined(__arm__) + __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); +#elif defined(__aarch64__) && defined(__APPLE__) + __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); +#elif defined(__aarch64__) + __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); +#elif defined(__powerpc64__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r13 is reserved for use as system thread ID by the Power 64-bit ABI. + register uintptr_t tp __asm__ ("r13"); + __asm__("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__powerpc__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r2 is reserved for use as system thread ID by the Power 32-bit ABI. + register uintptr_t tp __asm__ ("r2"); + __asm__ ("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__s390__) && defined(__GNUC__) + // Both GCC and Clang have supported __builtin_thread_pointer + // for s390 from long time ago. + tid = (uintptr_t)__builtin_thread_pointer(); +#else + # error "define _Py_ThreadId for this platform" +#endif + return tid; +} + +static inline Py_ALWAYS_INLINE int +_Py_IsOwnedByCurrentThread(PyObject *ob) +{ + return ob->ob_tid == _Py_ThreadId(); +} +#endif static inline Py_ssize_t Py_REFCNT(PyObject *ob) { +#if !defined(Py_GIL_DISABLED) return ob->ob_refcnt; +#else + uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return _Py_IMMORTAL_REFCNT; + } + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); + return _Py_STATIC_CAST(Py_ssize_t, local) + + Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); +#endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) @@ -223,10 +338,12 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) { -#if SIZEOF_VOID_P > 4 - return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; +#if defined(Py_GIL_DISABLED) + return (op->ob_ref_local == _Py_IMMORTAL_REFCNT_LOCAL); +#elif SIZEOF_VOID_P > 4 + return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); #else - return op->ob_refcnt == _Py_IMMORTAL_REFCNT; + return (op->ob_refcnt == _Py_IMMORTAL_REFCNT); #endif } #define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) @@ -239,7 +356,15 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { #endif +// Py_SET_REFCNT() implementation for stable ABI +PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); + static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 + // Stable ABI implements Py_SET_REFCNT() as a function call + // on limited C API version 3.13 and newer. + _Py_SetRefcnt(ob, refcnt); +#else // This immortal check is for code that is unaware of immortal objects. // The runtime tracks these objects and we should avoid as much // as possible having extensions inadvertently change the refcnt @@ -247,7 +372,33 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { if (_Py_IsImmortal(ob)) { return; } + +#ifndef Py_GIL_DISABLED ob->ob_refcnt = refcnt; +#else + if (_Py_IsOwnedByCurrentThread(ob)) { + if ((size_t)refcnt > (size_t)UINT32_MAX) { + // On overflow, make the object immortal + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + ob->ob_ref_shared = 0; + } + else { + // Set local refcount to desired refcount and shared refcount + // to zero, but preserve the shared refcount flags. + ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); + ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; + } + } + else { + // Set local refcount to zero and shared refcount to desired refcount. + // Mark the object as merged. + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = 0; + ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); + } +#endif // Py_GIL_DISABLED +#endif // Py_LIMITED_API+0 < 0x030d0000 } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) @@ -391,10 +542,20 @@ PyAPI_FUNC(PyObject *) PyObject_RichCompare(PyObject *, PyObject *, int); PyAPI_FUNC(int) PyObject_RichCompareBool(PyObject *, PyObject *, int); PyAPI_FUNC(PyObject *) PyObject_GetAttrString(PyObject *, const char *); PyAPI_FUNC(int) PyObject_SetAttrString(PyObject *, const char *, PyObject *); +PyAPI_FUNC(int) PyObject_DelAttrString(PyObject *v, const char *name); PyAPI_FUNC(int) PyObject_HasAttrString(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyObject_GetAttr(PyObject *, PyObject *); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PyObject_GetOptionalAttr(PyObject *, PyObject *, PyObject **); +PyAPI_FUNC(int) PyObject_GetOptionalAttrString(PyObject *, const char *, PyObject **); +#endif PyAPI_FUNC(int) PyObject_SetAttr(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(int) PyObject_DelAttr(PyObject *v, PyObject *name); PyAPI_FUNC(int) PyObject_HasAttr(PyObject *, PyObject *); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PyObject_HasAttrWithError(PyObject *, PyObject *); +PyAPI_FUNC(int) PyObject_HasAttrStringWithError(PyObject *, const char *); +#endif PyAPI_FUNC(PyObject *) PyObject_SelfIter(PyObject *); PyAPI_FUNC(PyObject *) PyObject_GenericGetAttr(PyObject *, PyObject *); PyAPI_FUNC(int) PyObject_GenericSetAttr(PyObject *, PyObject *, PyObject *); @@ -415,12 +576,6 @@ PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *); */ PyAPI_FUNC(PyObject *) PyObject_Dir(PyObject *); -/* Pickle support. */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyObject_GetState(PyObject *); -#endif - - /* Helpers for printing recursive container types */ PyAPI_FUNC(int) Py_ReprEnter(PyObject *); PyAPI_FUNC(void) Py_ReprLeave(PyObject *); @@ -585,20 +740,12 @@ decision that's up to the implementer of each new type so if you want, you can count such references to the type object.) */ -#ifdef Py_REF_DEBUG -# if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030A0000 -extern Py_ssize_t _Py_RefTotal; -# define _Py_INC_REFTOTAL() _Py_RefTotal++ -# define _Py_DEC_REFTOTAL() _Py_RefTotal-- -# elif !defined(Py_LIMITED_API) || Py_LIMITED_API+0 > 0x030C0000 -PyAPI_FUNC(void) _Py_IncRefTotal_DO_NOT_USE_THIS(void); -PyAPI_FUNC(void) _Py_DecRefTotal_DO_NOT_USE_THIS(void); -# define _Py_INC_REFTOTAL() _Py_IncRefTotal_DO_NOT_USE_THIS() -# define _Py_DEC_REFTOTAL() _Py_DecRefTotal_DO_NOT_USE_THIS() -# endif +#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op); -#endif /* Py_REF_DEBUG */ +PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); +PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); +#endif // Py_REF_DEBUG && !Py_LIMITED_API PyAPI_FUNC(void) _Py_Dealloc(PyObject *); @@ -616,17 +763,39 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *); static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { -#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000 - // Stable ABI for Python 3.10 built in debug mode. +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) + // Stable ABI implements Py_INCREF() as a function call on limited C API + // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() + // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. +# if Py_LIMITED_API+0 >= 0x030a00A7 _Py_IncRef(op); +# else + Py_IncRef(op); +# endif #else // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. -#if SIZEOF_VOID_P > 4 +#if defined(Py_GIL_DISABLED) + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + uint32_t new_local = local + 1; + if (new_local == 0) { + // local is equal to _Py_IMMORTAL_REFCNT: do nothing + return; + } + if (_Py_IsOwnedByCurrentThread(op)) { + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local); + } + else { + _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); + } +#elif SIZEOF_VOID_P > 4 // Portable saturated add, branching on the carry flag and set low bits PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; PY_UINT32_T new_refcnt = cur_refcnt + 1; if (new_refcnt == 0) { + // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, + // do nothing return; } op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; @@ -639,7 +808,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) #endif _Py_INCREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_INC_REFTOTAL(); + _Py_INCREF_IncRefTotal(); #endif #endif } @@ -647,27 +816,91 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) # define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) #endif -#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000 -// Stable ABI for limited C API version 3.10 of Python debug build + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +// Implements Py_DECREF on objects not owned by the current thread. +PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); +PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); + +// Called from Py_DECREF by the owning thread when the local refcount reaches +// zero. The call will deallocate the object if the shared refcount is also +// zero. Otherwise, the thread gives up ownership and merges the reference +// count fields. +PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); +#endif + +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +// Stable ABI implements Py_DECREF() as a function call on limited C API +// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was +// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. static inline void Py_DECREF(PyObject *op) { +# if Py_LIMITED_API+0 >= 0x030a00A7 _Py_DecRef(op); +# else + Py_DecRef(op); +# endif } #define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) -#elif defined(Py_REF_DEBUG) +#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { - if (_Py_IsImmortal(op)) { + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { return; } _Py_DECREF_STAT_INC(); - _Py_DEC_REFTOTAL(); - if (--op->ob_refcnt != 0) { - if (op->ob_refcnt < 0) { + _Py_DECREF_DecRefTotal(); + if (_Py_IsOwnedByCurrentThread(op)) { + if (local == 0) { _Py_NegativeRefcount(filename, lineno, op); } + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } } else { + _Py_DecRefSharedDebug(op, filename, lineno); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) +static inline void Py_DECREF(PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return; + } + _Py_DECREF_STAT_INC(); + if (_Py_IsOwnedByCurrentThread(op)) { + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefShared(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ + if (op->ob_refcnt <= 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (--op->ob_refcnt == 0) { _Py_Dealloc(op); } } @@ -689,9 +922,6 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) #define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) #endif -#undef _Py_INC_REFTOTAL -#undef _Py_DEC_REFTOTAL - /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear * and tp_dealloc implementations. @@ -820,8 +1050,6 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) /* _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). - -Don't forget to apply Py_INCREF() when returning this value!!! */ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ #define Py_None (&_Py_NoneStruct) diff --git a/Include/objimpl.h b/Include/objimpl.h index ef871c5ea93ebe..ff5fa7a8c1d3d8 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -1,12 +1,8 @@ -/* The PyObject_ memory family: high-level object memory interfaces. - See pymem.h for the low-level PyMem_ family. -*/ +// The PyObject_ memory family: high-level object memory interfaces. +// See pymem.h for the low-level PyMem_ family. #ifndef Py_OBJIMPL_H #define Py_OBJIMPL_H - -#include "pymem.h" - #ifdef __cplusplus extern "C" { #endif @@ -39,7 +35,7 @@ Functions and macros for modules that implement new object types. fields, this also fills in the ob_size field. - PyObject_Free(op) releases the memory allocated for an object. It does not - run a destructor -- it only frees the memory. PyObject_Free is identical. + run a destructor -- it only frees the memory. - PyObject_Init(op, typeobj) and PyObject_InitVar(op, typeobj, n) don't allocate memory. Instead of a 'type' parameter, they take a pointer to a @@ -231,4 +227,4 @@ PyAPI_FUNC(int) PyObject_GC_IsFinalized(PyObject *); #ifdef __cplusplus } #endif -#endif /* !Py_OBJIMPL_H */ +#endif // !Py_OBJIMPL_H diff --git a/Include/opcode.h b/Include/opcode.h index c3b6e5dce49ad3..2619b690019acc 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -1,234 +1,11 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py - #ifndef Py_OPCODE_H #define Py_OPCODE_H #ifdef __cplusplus extern "C" { #endif +#include "opcode_ids.h" -/* Instruction opcodes for compiled code */ -#define CACHE 0 -#define POP_TOP 1 -#define PUSH_NULL 2 -#define INTERPRETER_EXIT 3 -#define END_FOR 4 -#define END_SEND 5 -#define NOP 9 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define RESERVED 17 -#define BINARY_SUBSCR 25 -#define BINARY_SLICE 26 -#define STORE_SLICE 27 -#define GET_LEN 30 -#define MATCH_MAPPING 31 -#define MATCH_SEQUENCE 32 -#define MATCH_KEYS 33 -#define PUSH_EXC_INFO 35 -#define CHECK_EXC_MATCH 36 -#define CHECK_EG_MATCH 37 -#define WITH_EXCEPT_START 49 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define BEFORE_WITH 53 -#define END_ASYNC_FOR 54 -#define CLEANUP_THROW 55 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define LOAD_BUILD_CLASS 71 -#define LOAD_ASSERTION_ERROR 74 -#define RETURN_GENERATOR 75 -#define RETURN_VALUE 83 -#define SETUP_ANNOTATIONS 85 -#define POP_EXCEPT 89 -#define HAVE_ARGUMENT 90 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define SWAP 99 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define IS_OP 117 -#define CONTAINS_OP 118 -#define RERAISE 119 -#define COPY 120 -#define RETURN_CONST 121 -#define BINARY_OP 122 -#define SEND 123 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define LOAD_FAST_CHECK 127 -#define POP_JUMP_IF_NOT_NONE 128 -#define POP_JUMP_IF_NONE 129 -#define RAISE_VARARGS 130 -#define GET_AWAITABLE 131 -#define MAKE_FUNCTION 132 -#define BUILD_SLICE 133 -#define JUMP_BACKWARD_NO_INTERRUPT 134 -#define MAKE_CELL 135 -#define LOAD_CLOSURE 136 -#define LOAD_DEREF 137 -#define STORE_DEREF 138 -#define DELETE_DEREF 139 -#define JUMP_BACKWARD 140 -#define LOAD_SUPER_ATTR 141 -#define CALL_FUNCTION_EX 142 -#define LOAD_FAST_AND_CLEAR 143 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define LOAD_CLASSDEREF 148 -#define COPY_FREE_VARS 149 -#define YIELD_VALUE 150 -#define RESUME 151 -#define MATCH_CLASS 152 -#define FORMAT_VALUE 155 -#define BUILD_CONST_KEY_MAP 156 -#define BUILD_STRING 157 -#define LIST_EXTEND 162 -#define SET_UPDATE 163 -#define DICT_MERGE 164 -#define DICT_UPDATE 165 -#define CALL 171 -#define KW_NAMES 172 -#define CALL_INTRINSIC_1 173 -#define CALL_INTRINSIC_2 174 -#define MIN_INSTRUMENTED_OPCODE 238 -#define INSTRUMENTED_POP_JUMP_IF_NONE 238 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 -#define INSTRUMENTED_RESUME 240 -#define INSTRUMENTED_CALL 241 -#define INSTRUMENTED_RETURN_VALUE 242 -#define INSTRUMENTED_YIELD_VALUE 243 -#define INSTRUMENTED_CALL_FUNCTION_EX 244 -#define INSTRUMENTED_JUMP_FORWARD 245 -#define INSTRUMENTED_JUMP_BACKWARD 246 -#define INSTRUMENTED_RETURN_CONST 247 -#define INSTRUMENTED_FOR_ITER 248 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_END_FOR 251 -#define INSTRUMENTED_END_SEND 252 -#define INSTRUMENTED_INSTRUCTION 253 -#define INSTRUMENTED_LINE 254 -#define MIN_PSEUDO_OPCODE 256 -#define SETUP_FINALLY 256 -#define SETUP_CLEANUP 257 -#define SETUP_WITH 258 -#define POP_BLOCK 259 -#define JUMP 260 -#define JUMP_NO_INTERRUPT 261 -#define LOAD_METHOD 262 -#define LOAD_SUPER_METHOD 263 -#define LOAD_ZERO_SUPER_METHOD 264 -#define LOAD_ZERO_SUPER_ATTR 265 -#define STORE_FAST_MAYBE_NULL 266 -#define MAX_PSEUDO_OPCODE 266 -#define BINARY_OP_ADD_FLOAT 6 -#define BINARY_OP_ADD_INT 7 -#define BINARY_OP_ADD_UNICODE 8 -#define BINARY_OP_INPLACE_ADD_UNICODE 10 -#define BINARY_OP_MULTIPLY_FLOAT 13 -#define BINARY_OP_MULTIPLY_INT 14 -#define BINARY_OP_SUBTRACT_FLOAT 16 -#define BINARY_OP_SUBTRACT_INT 18 -#define BINARY_SUBSCR_DICT 19 -#define BINARY_SUBSCR_GETITEM 20 -#define BINARY_SUBSCR_LIST_INT 21 -#define BINARY_SUBSCR_TUPLE_INT 22 -#define CALL_PY_EXACT_ARGS 23 -#define CALL_PY_WITH_DEFAULTS 24 -#define CALL_BOUND_METHOD_EXACT_ARGS 28 -#define CALL_BUILTIN_CLASS 29 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 -#define CALL_NO_KW_BUILTIN_FAST 39 -#define CALL_NO_KW_BUILTIN_O 40 -#define CALL_NO_KW_ISINSTANCE 41 -#define CALL_NO_KW_LEN 42 -#define CALL_NO_KW_LIST_APPEND 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 -#define CALL_NO_KW_STR_1 47 -#define CALL_NO_KW_TUPLE_1 48 -#define CALL_NO_KW_TYPE_1 56 -#define COMPARE_OP_FLOAT 57 -#define COMPARE_OP_INT 58 -#define COMPARE_OP_STR 59 -#define FOR_ITER_LIST 62 -#define FOR_ITER_TUPLE 63 -#define FOR_ITER_RANGE 64 -#define FOR_ITER_GEN 65 -#define LOAD_SUPER_ATTR_ATTR 66 -#define LOAD_SUPER_ATTR_METHOD 67 -#define LOAD_ATTR_CLASS 70 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 72 -#define LOAD_ATTR_INSTANCE_VALUE 73 -#define LOAD_ATTR_MODULE 76 -#define LOAD_ATTR_PROPERTY 77 -#define LOAD_ATTR_SLOT 78 -#define LOAD_ATTR_WITH_HINT 79 -#define LOAD_ATTR_METHOD_LAZY_DICT 80 -#define LOAD_ATTR_METHOD_NO_DICT 81 -#define LOAD_ATTR_METHOD_WITH_VALUES 82 -#define LOAD_CONST__LOAD_FAST 84 -#define LOAD_FAST__LOAD_CONST 86 -#define LOAD_FAST__LOAD_FAST 87 -#define LOAD_GLOBAL_BUILTIN 88 -#define LOAD_GLOBAL_MODULE 111 -#define STORE_ATTR_INSTANCE_VALUE 112 -#define STORE_ATTR_SLOT 113 -#define STORE_ATTR_WITH_HINT 153 -#define STORE_FAST__LOAD_FAST 154 -#define STORE_FAST__STORE_FAST 158 -#define STORE_SUBSCR_DICT 159 -#define STORE_SUBSCR_LIST_INT 160 -#define UNPACK_SEQUENCE_LIST 161 -#define UNPACK_SEQUENCE_TUPLE 166 -#define UNPACK_SEQUENCE_TWO_TUPLE 167 -#define SEND_GEN 168 - -#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ - || ((op) == JUMP) \ - || ((op) == JUMP_NO_INTERRUPT) \ - || ((op) == LOAD_METHOD) \ - || ((op) == LOAD_SUPER_METHOD) \ - || ((op) == LOAD_ZERO_SUPER_METHOD) \ - || ((op) == LOAD_ZERO_SUPER_ATTR) \ - || ((op) == STORE_FAST_MAYBE_NULL) \ - ) - -#define HAS_CONST(op) (false\ - || ((op) == LOAD_CONST) \ - || ((op) == RETURN_CONST) \ - || ((op) == KW_NAMES) \ - ) #define NB_ADD 0 #define NB_AND 1 @@ -257,10 +34,7 @@ extern "C" { #define NB_INPLACE_TRUE_DIVIDE 24 #define NB_INPLACE_XOR 25 -/* Defined in Lib/opcode.py */ -#define ENABLE_SPECIALIZATION 1 - -#define IS_PSEUDO_OPCODE(op) (((op) >= MIN_PSEUDO_OPCODE) && ((op) <= MAX_PSEUDO_OPCODE)) +#define NB_OPARG_LAST 25 #ifdef __cplusplus } diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h new file mode 100644 index 00000000000000..47f809e345f61c --- /dev/null +++ b/Include/opcode_ids.h @@ -0,0 +1,240 @@ +// This file is generated by Tools/cases_generator/opcode_id_generator.py +// from: +// ['./Python/bytecodes.c'] +// Do not edit! + +#ifndef Py_OPCODE_IDS_H +#define Py_OPCODE_IDS_H +#ifdef __cplusplus +extern "C" { +#endif + +/* Instruction opcodes for compiled code */ +#define CACHE 0 +#define BEFORE_ASYNC_WITH 1 +#define BEFORE_WITH 2 +#define BINARY_OP_INPLACE_ADD_UNICODE 3 +#define BINARY_SLICE 4 +#define BINARY_SUBSCR 5 +#define CHECK_EG_MATCH 6 +#define CHECK_EXC_MATCH 7 +#define CLEANUP_THROW 8 +#define DELETE_SUBSCR 9 +#define END_ASYNC_FOR 10 +#define END_FOR 11 +#define END_SEND 12 +#define EXIT_INIT_CHECK 13 +#define FORMAT_SIMPLE 14 +#define FORMAT_WITH_SPEC 15 +#define GET_AITER 16 +#define RESERVED 17 +#define GET_ANEXT 18 +#define GET_ITER 19 +#define GET_LEN 20 +#define GET_YIELD_FROM_ITER 21 +#define INTERPRETER_EXIT 22 +#define LOAD_ASSERTION_ERROR 23 +#define LOAD_BUILD_CLASS 24 +#define LOAD_LOCALS 25 +#define MAKE_FUNCTION 26 +#define MATCH_KEYS 27 +#define MATCH_MAPPING 28 +#define MATCH_SEQUENCE 29 +#define NOP 30 +#define POP_EXCEPT 31 +#define POP_TOP 32 +#define PUSH_EXC_INFO 33 +#define PUSH_NULL 34 +#define RETURN_GENERATOR 35 +#define RETURN_VALUE 36 +#define SETUP_ANNOTATIONS 37 +#define STORE_SLICE 38 +#define STORE_SUBSCR 39 +#define TO_BOOL 40 +#define UNARY_INVERT 41 +#define UNARY_NEGATIVE 42 +#define UNARY_NOT 43 +#define WITH_EXCEPT_START 44 +#define BINARY_OP 45 +#define BUILD_CONST_KEY_MAP 46 +#define BUILD_LIST 47 +#define BUILD_MAP 48 +#define BUILD_SET 49 +#define BUILD_SLICE 50 +#define BUILD_STRING 51 +#define BUILD_TUPLE 52 +#define CALL 53 +#define CALL_FUNCTION_EX 54 +#define CALL_INTRINSIC_1 55 +#define CALL_INTRINSIC_2 56 +#define CALL_KW 57 +#define COMPARE_OP 58 +#define CONTAINS_OP 59 +#define CONVERT_VALUE 60 +#define COPY 61 +#define COPY_FREE_VARS 62 +#define DELETE_ATTR 63 +#define DELETE_DEREF 64 +#define DELETE_FAST 65 +#define DELETE_GLOBAL 66 +#define DELETE_NAME 67 +#define DICT_MERGE 68 +#define DICT_UPDATE 69 +#define ENTER_EXECUTOR 70 +#define EXTENDED_ARG 71 +#define FOR_ITER 72 +#define GET_AWAITABLE 73 +#define IMPORT_FROM 74 +#define IMPORT_NAME 75 +#define IS_OP 76 +#define JUMP_BACKWARD 77 +#define JUMP_BACKWARD_NO_INTERRUPT 78 +#define JUMP_FORWARD 79 +#define LIST_APPEND 80 +#define LIST_EXTEND 81 +#define LOAD_ATTR 82 +#define LOAD_CONST 83 +#define LOAD_DEREF 84 +#define LOAD_FAST 85 +#define LOAD_FAST_AND_CLEAR 86 +#define LOAD_FAST_CHECK 87 +#define LOAD_FAST_LOAD_FAST 88 +#define LOAD_FROM_DICT_OR_DEREF 89 +#define LOAD_FROM_DICT_OR_GLOBALS 90 +#define LOAD_GLOBAL 91 +#define LOAD_NAME 92 +#define LOAD_SUPER_ATTR 93 +#define MAKE_CELL 94 +#define MAP_ADD 95 +#define MATCH_CLASS 96 +#define POP_JUMP_IF_FALSE 97 +#define POP_JUMP_IF_NONE 98 +#define POP_JUMP_IF_NOT_NONE 99 +#define POP_JUMP_IF_TRUE 100 +#define RAISE_VARARGS 101 +#define RERAISE 102 +#define RETURN_CONST 103 +#define SEND 104 +#define SET_ADD 105 +#define SET_FUNCTION_ATTRIBUTE 106 +#define SET_UPDATE 107 +#define STORE_ATTR 108 +#define STORE_DEREF 109 +#define STORE_FAST 110 +#define STORE_FAST_LOAD_FAST 111 +#define STORE_FAST_STORE_FAST 112 +#define STORE_GLOBAL 113 +#define STORE_NAME 114 +#define SWAP 115 +#define UNPACK_EX 116 +#define UNPACK_SEQUENCE 117 +#define YIELD_VALUE 118 +#define RESUME 149 +#define BINARY_OP_ADD_FLOAT 150 +#define BINARY_OP_ADD_INT 151 +#define BINARY_OP_ADD_UNICODE 152 +#define BINARY_OP_MULTIPLY_FLOAT 153 +#define BINARY_OP_MULTIPLY_INT 154 +#define BINARY_OP_SUBTRACT_FLOAT 155 +#define BINARY_OP_SUBTRACT_INT 156 +#define BINARY_SUBSCR_DICT 157 +#define BINARY_SUBSCR_GETITEM 158 +#define BINARY_SUBSCR_LIST_INT 159 +#define BINARY_SUBSCR_STR_INT 160 +#define BINARY_SUBSCR_TUPLE_INT 161 +#define CALL_ALLOC_AND_ENTER_INIT 162 +#define CALL_BOUND_METHOD_EXACT_ARGS 163 +#define CALL_BUILTIN_CLASS 164 +#define CALL_BUILTIN_FAST 165 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 166 +#define CALL_BUILTIN_O 167 +#define CALL_ISINSTANCE 168 +#define CALL_LEN 169 +#define CALL_LIST_APPEND 170 +#define CALL_METHOD_DESCRIPTOR_FAST 171 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 172 +#define CALL_METHOD_DESCRIPTOR_NOARGS 173 +#define CALL_METHOD_DESCRIPTOR_O 174 +#define CALL_PY_EXACT_ARGS 175 +#define CALL_PY_WITH_DEFAULTS 176 +#define CALL_STR_1 177 +#define CALL_TUPLE_1 178 +#define CALL_TYPE_1 179 +#define COMPARE_OP_FLOAT 180 +#define COMPARE_OP_INT 181 +#define COMPARE_OP_STR 182 +#define FOR_ITER_GEN 183 +#define FOR_ITER_LIST 184 +#define FOR_ITER_RANGE 185 +#define FOR_ITER_TUPLE 186 +#define LOAD_ATTR_CLASS 187 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 188 +#define LOAD_ATTR_INSTANCE_VALUE 189 +#define LOAD_ATTR_METHOD_LAZY_DICT 190 +#define LOAD_ATTR_METHOD_NO_DICT 191 +#define LOAD_ATTR_METHOD_WITH_VALUES 192 +#define LOAD_ATTR_MODULE 193 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 194 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 195 +#define LOAD_ATTR_PROPERTY 196 +#define LOAD_ATTR_SLOT 197 +#define LOAD_ATTR_WITH_HINT 198 +#define LOAD_GLOBAL_BUILTIN 199 +#define LOAD_GLOBAL_MODULE 200 +#define LOAD_SUPER_ATTR_ATTR 201 +#define LOAD_SUPER_ATTR_METHOD 202 +#define RESUME_CHECK 203 +#define SEND_GEN 204 +#define STORE_ATTR_INSTANCE_VALUE 205 +#define STORE_ATTR_SLOT 206 +#define STORE_ATTR_WITH_HINT 207 +#define STORE_SUBSCR_DICT 208 +#define STORE_SUBSCR_LIST_INT 209 +#define TO_BOOL_ALWAYS_TRUE 210 +#define TO_BOOL_BOOL 211 +#define TO_BOOL_INT 212 +#define TO_BOOL_LIST 213 +#define TO_BOOL_NONE 214 +#define TO_BOOL_STR 215 +#define UNPACK_SEQUENCE_LIST 216 +#define UNPACK_SEQUENCE_TUPLE 217 +#define UNPACK_SEQUENCE_TWO_TUPLE 218 +#define INSTRUMENTED_RESUME 236 +#define INSTRUMENTED_END_FOR 237 +#define INSTRUMENTED_END_SEND 238 +#define INSTRUMENTED_RETURN_VALUE 239 +#define INSTRUMENTED_RETURN_CONST 240 +#define INSTRUMENTED_YIELD_VALUE 241 +#define INSTRUMENTED_LOAD_SUPER_ATTR 242 +#define INSTRUMENTED_FOR_ITER 243 +#define INSTRUMENTED_CALL 244 +#define INSTRUMENTED_CALL_KW 245 +#define INSTRUMENTED_CALL_FUNCTION_EX 246 +#define INSTRUMENTED_INSTRUCTION 247 +#define INSTRUMENTED_JUMP_FORWARD 248 +#define INSTRUMENTED_JUMP_BACKWARD 249 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 251 +#define INSTRUMENTED_POP_JUMP_IF_NONE 252 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 +#define INSTRUMENTED_LINE 254 +#define JUMP 256 +#define JUMP_NO_INTERRUPT 257 +#define LOAD_CLOSURE 258 +#define LOAD_METHOD 259 +#define LOAD_SUPER_METHOD 260 +#define LOAD_ZERO_SUPER_ATTR 261 +#define LOAD_ZERO_SUPER_METHOD 262 +#define POP_BLOCK 263 +#define SETUP_CLEANUP 264 +#define SETUP_FINALLY 265 +#define SETUP_WITH 266 +#define STORE_FAST_MAYBE_NULL 267 + +#define HAVE_ARGUMENT 45 +#define MIN_INSTRUMENTED_OPCODE 236 + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPCODE_IDS_H */ diff --git a/Include/osdefs.h b/Include/osdefs.h index 3243944a1483e9..2599e87a9d7c4b 100644 --- a/Include/osdefs.h +++ b/Include/osdefs.h @@ -1,51 +1,57 @@ +// Operating system dependencies. +// +// Define constants: +// +// - ALTSEP +// - DELIM +// - MAXPATHLEN +// - SEP + #ifndef Py_OSDEFS_H #define Py_OSDEFS_H #ifdef __cplusplus extern "C" { #endif - -/* Operating system dependencies */ - #ifdef MS_WINDOWS -#define SEP L'\\' -#define ALTSEP L'/' -#define MAXPATHLEN 256 -#define DELIM L';' +# define SEP L'\\' +# define ALTSEP L'/' +# define MAXPATHLEN 256 +# define DELIM L';' #endif #ifdef __VXWORKS__ -#define DELIM L';' +# define DELIM L';' #endif /* Filename separator */ #ifndef SEP -#define SEP L'/' +# define SEP L'/' #endif /* Max pathname length */ #ifdef __hpux -#include -#include -#ifndef PATH_MAX -#define PATH_MAX MAXPATHLEN -#endif +# include +# include +# ifndef PATH_MAX +# define PATH_MAX MAXPATHLEN +# endif #endif #ifndef MAXPATHLEN -#if defined(PATH_MAX) && PATH_MAX > 1024 -#define MAXPATHLEN PATH_MAX -#else -#define MAXPATHLEN 1024 -#endif +# if defined(PATH_MAX) && PATH_MAX > 1024 +# define MAXPATHLEN PATH_MAX +# else +# define MAXPATHLEN 1024 +# endif #endif /* Search path entry delimiter */ #ifndef DELIM -#define DELIM L':' +# define DELIM L':' #endif #ifdef __cplusplus } #endif -#endif /* !Py_OSDEFS_H */ +#endif // !Py_OSDEFS_H diff --git a/Include/patchlevel.h b/Include/patchlevel.h index aaedd563a905e9..fad79ecfda7b28 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 12 +#define PY_MINOR_VERSION 13 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 7 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.12.0a7+" +#define PY_VERSION "3.13.0a2+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pyatomic.h b/Include/pyatomic.h new file mode 100644 index 00000000000000..2ce2c81cf5251a --- /dev/null +++ b/Include/pyatomic.h @@ -0,0 +1,16 @@ +#ifndef Py_ATOMIC_H +#define Py_ATOMIC_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_ATOMIC_H +# include "cpython/pyatomic.h" +# undef Py_CPYTHON_ATOMIC_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_ATOMIC_H */ diff --git a/Include/pycapsule.h b/Include/pycapsule.h index 929a9a685259fb..666b9f86739670 100644 --- a/Include/pycapsule.h +++ b/Include/pycapsule.h @@ -52,7 +52,6 @@ PyAPI_FUNC(void *) PyCapsule_Import( const char *name, /* UTF-8 encoded string */ int no_block); - #ifdef __cplusplus } #endif diff --git a/Include/pyerrors.h b/Include/pyerrors.h index d089fa71779330..5d0028c116e2d8 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -1,13 +1,11 @@ +// Error handling definitions + #ifndef Py_ERRORS_H #define Py_ERRORS_H #ifdef __cplusplus extern "C" { #endif -#include // va_list - -/* Error handling definitions */ - PyAPI_FUNC(void) PyErr_SetNone(PyObject *); PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *); PyAPI_FUNC(void) PyErr_SetString( diff --git a/Include/pyhash.h b/Include/pyhash.h index 182d223fab1cac..3e23e2758808d7 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -1,100 +1,10 @@ #ifndef Py_HASH_H - #define Py_HASH_H #ifdef __cplusplus extern "C" { #endif -/* Helpers for hash functions */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); -PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); -// Similar to _Py_HashPointer(), but don't replace -1 with -2 -PyAPI_FUNC(Py_hash_t) _Py_HashPointerRaw(const void*); -PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); -#endif - -/* Prime multiplier used in string and various other hashes. */ -#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ - -/* Parameters used for the numeric hash implementation. See notes for - _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ - -#if SIZEOF_VOID_P >= 8 -# define _PyHASH_BITS 61 -#else -# define _PyHASH_BITS 31 -#endif - -#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) -#define _PyHASH_INF 314159 -#define _PyHASH_IMAG _PyHASH_MULTIPLIER - - -/* hash secret - * - * memory layout on 64 bit systems - * cccccccc cccccccc cccccccc uc -- unsigned char[24] - * pppppppp ssssssss ........ fnv -- two Py_hash_t - * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t - * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeeeeeee pyexpat XML hash salt - * - * memory layout on 32 bit systems - * cccccccc cccccccc cccccccc uc - * ppppssss ........ ........ fnv -- two Py_hash_t - * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) - * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeee.... pyexpat XML hash salt - * - * (*) The siphash member may not be available on 32 bit platforms without - * an unsigned int64 data type. - */ -#ifndef Py_LIMITED_API -typedef union { - /* ensure 24 bytes */ - unsigned char uc[24]; - /* two Py_hash_t for FNV */ - struct { - Py_hash_t prefix; - Py_hash_t suffix; - } fnv; - /* two uint64 for SipHash24 */ - struct { - uint64_t k0; - uint64_t k1; - } siphash; - /* a different (!) Py_hash_t for small string optimization */ - struct { - unsigned char padding[16]; - Py_hash_t suffix; - } djbx33a; - struct { - unsigned char padding[16]; - Py_hash_t hashsalt; - } expat; -} _Py_HashSecret_t; -PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; - -#ifdef Py_DEBUG -PyAPI_DATA(int) _Py_HashSecret_Initialized; -#endif - - -/* hash function definition */ -typedef struct { - Py_hash_t (*const hash)(const void *, Py_ssize_t); - const char *name; - const int hash_bits; - const int seed_bits; -} PyHash_FuncDef; - -PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); -#endif - - -/* cutoff for small string DJBX33A optimization in range [1, cutoff). +/* Cutoff for small string DJBX33A optimization in range [1, cutoff). * * About 50% of the strings in a typical Python application are smaller than * 6 to 7 chars. However DJBX33A is vulnerable to hash collision attacks. @@ -112,7 +22,7 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); #endif /* Py_HASH_CUTOFF */ -/* hash algorithm selection +/* Hash algorithm selection * * The values for Py_HASH_* are hard-coded in the * configure script. @@ -137,8 +47,13 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); # endif /* uint64_t && uint32_t && aligned */ #endif /* Py_HASH_ALGORITHM */ +#ifndef Py_LIMITED_API +# define Py_CPYTHON_HASH_H +# include "cpython/pyhash.h" +# undef Py_CPYTHON_HASH_H +#endif + #ifdef __cplusplus } #endif - -#endif /* !Py_HASH_H */ +#endif // !Py_HASH_H diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index e4c3b09c963fe8..c1e2bc5e323358 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -34,18 +34,12 @@ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); /* In pathconfig.c */ -Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); - -Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); - -PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); - -PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); -PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); -PyAPI_FUNC(wchar_t *) Py_GetPath(void); -Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetPath(const wchar_t *); +Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); +Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); +Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); +Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); +Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); +Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPath(void); #ifdef MS_WINDOWS int _Py_CheckPython3(void); #endif @@ -66,6 +60,10 @@ PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t); PyAPI_DATA(const unsigned long) Py_Version; #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 +PyAPI_FUNC(int) Py_IsFinalizing(void); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYLIFECYCLE_H # include "cpython/pylifecycle.h" diff --git a/Include/pymacconfig.h b/Include/pymacconfig.h index 00459a03b980be..615abe103ca038 100644 --- a/Include/pymacconfig.h +++ b/Include/pymacconfig.h @@ -1,90 +1,91 @@ -#ifndef PYMACCONFIG_H -#define PYMACCONFIG_H - /* - * This file moves some of the autoconf magic to compile-time - * when building on MacOSX. This is needed for building 4-way - * universal binaries and for 64-bit universal binaries because - * the values redefined below aren't configure-time constant but - * only compile-time constant in these scenarios. - */ +// This file moves some of the autoconf magic to compile-time when building on +// macOS. This is needed for building 4-way universal binaries and for 64-bit +// universal binaries because the values redefined below aren't configure-time +// constant but only compile-time constant in these scenarios. -#if defined(__APPLE__) +#ifndef PY_MACCONFIG_H +#define PY_MACCONFIG_H +#ifdef __APPLE__ -# undef SIZEOF_LONG -# undef SIZEOF_PTHREAD_T -# undef SIZEOF_SIZE_T -# undef SIZEOF_TIME_T -# undef SIZEOF_VOID_P -# undef SIZEOF__BOOL -# undef SIZEOF_UINTPTR_T -# undef SIZEOF_PTHREAD_T -# undef WORDS_BIGENDIAN -# undef DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 -# undef DOUBLE_IS_BIG_ENDIAN_IEEE754 -# undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 -# undef HAVE_GCC_ASM_FOR_X87 +#undef ALIGNOF_MAX_ALIGN_T +#undef SIZEOF_LONG +#undef SIZEOF_LONG_DOUBLE +#undef SIZEOF_PTHREAD_T +#undef SIZEOF_SIZE_T +#undef SIZEOF_TIME_T +#undef SIZEOF_VOID_P +#undef SIZEOF__BOOL +#undef SIZEOF_UINTPTR_T +#undef SIZEOF_PTHREAD_T +#undef WORDS_BIGENDIAN +#undef DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 +#undef DOUBLE_IS_BIG_ENDIAN_IEEE754 +#undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 +#undef HAVE_GCC_ASM_FOR_X87 +#undef HAVE_GCC_ASM_FOR_X64 -# undef VA_LIST_IS_ARRAY -# if defined(__LP64__) && defined(__x86_64__) -# define VA_LIST_IS_ARRAY 1 -# endif - -# undef HAVE_LARGEFILE_SUPPORT -# ifndef __LP64__ -# define HAVE_LARGEFILE_SUPPORT 1 -# endif - -# undef SIZEOF_LONG -# ifdef __LP64__ -# define SIZEOF__BOOL 1 -# define SIZEOF__BOOL 1 -# define SIZEOF_LONG 8 -# define SIZEOF_PTHREAD_T 8 -# define SIZEOF_SIZE_T 8 -# define SIZEOF_TIME_T 8 -# define SIZEOF_VOID_P 8 -# define SIZEOF_UINTPTR_T 8 -# define SIZEOF_PTHREAD_T 8 -# else -# ifdef __ppc__ -# define SIZEOF__BOOL 4 -# else -# define SIZEOF__BOOL 1 -# endif -# define SIZEOF_LONG 4 -# define SIZEOF_PTHREAD_T 4 -# define SIZEOF_SIZE_T 4 -# define SIZEOF_TIME_T 4 -# define SIZEOF_VOID_P 4 -# define SIZEOF_UINTPTR_T 4 -# define SIZEOF_PTHREAD_T 4 -# endif +#undef VA_LIST_IS_ARRAY +#if defined(__LP64__) && defined(__x86_64__) +# define VA_LIST_IS_ARRAY 1 +#endif -# if defined(__LP64__) - /* MacOSX 10.4 (the first release to support 64-bit code - * at all) only supports 64-bit in the UNIX layer. - * Therefore suppress the toolbox-glue in 64-bit mode. - */ +#undef HAVE_LARGEFILE_SUPPORT +#ifndef __LP64__ +# define HAVE_LARGEFILE_SUPPORT 1 +#endif - /* In 64-bit mode setpgrp always has no arguments, in 32-bit - * mode that depends on the compilation environment - */ -# undef SETPGRP_HAVE_ARG +#undef SIZEOF_LONG +#ifdef __LP64__ +# define SIZEOF__BOOL 1 +# define SIZEOF__BOOL 1 +# define SIZEOF_LONG 8 +# define SIZEOF_PTHREAD_T 8 +# define SIZEOF_SIZE_T 8 +# define SIZEOF_TIME_T 8 +# define SIZEOF_VOID_P 8 +# define SIZEOF_UINTPTR_T 8 +# define SIZEOF_PTHREAD_T 8 +#else +# ifdef __ppc__ +# define SIZEOF__BOOL 4 +# else +# define SIZEOF__BOOL 1 +# endif +# define SIZEOF_LONG 4 +# define SIZEOF_PTHREAD_T 4 +# define SIZEOF_SIZE_T 4 +# define SIZEOF_TIME_T 4 +# define SIZEOF_VOID_P 4 +# define SIZEOF_UINTPTR_T 4 +# define SIZEOF_PTHREAD_T 4 +#endif -# endif +// macOS 10.4 (the first release to support 64-bit code +// at all) only supports 64-bit in the UNIX layer. +// Therefore suppress the toolbox-glue in 64-bit mode. +// +// In 64-bit mode setpgrp always has no arguments, in 32-bit +// mode that depends on the compilation environment +#if defined(__LP64__) +# undef SETPGRP_HAVE_ARG +#endif #ifdef __BIG_ENDIAN__ -#define WORDS_BIGENDIAN 1 -#define DOUBLE_IS_BIG_ENDIAN_IEEE754 +# define WORDS_BIGENDIAN 1 +# define DOUBLE_IS_BIG_ENDIAN_IEEE754 #else -#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 -#endif /* __BIG_ENDIAN */ - -#ifdef __i386__ -# define HAVE_GCC_ASM_FOR_X87 +# define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #endif +#if defined(__i386__) || defined(__x86_64__) +# define HAVE_GCC_ASM_FOR_X87 +# define ALIGNOF_MAX_ALIGN_T 16 +# define HAVE_GCC_ASM_FOR_X64 1 +# define SIZEOF_LONG_DOUBLE 16 +#else +# define ALIGNOF_MAX_ALIGN_T 8 +# define SIZEOF_LONG_DOUBLE 8 +#endif -#endif /* defined(_APPLE__) */ - -#endif /* PYMACCONFIG_H */ +#endif // __APPLE__ +#endif // !PY_MACCONFIG_H diff --git a/Include/pymacro.h b/Include/pymacro.h index 342d2a7b844adf..9d264fe6eea1d4 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -118,6 +118,15 @@ */ #if defined(__GNUC__) || defined(__clang__) # define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) +#elif defined(_MSC_VER) + // Disable warning C4100: unreferenced formal parameter, + // declare the parameter, + // restore old compiler warnings. +# define Py_UNUSED(name) \ + __pragma(warning(push)) \ + __pragma(warning(suppress: 4100)) \ + _unused_ ## name \ + __pragma(warning(pop)) #else # define Py_UNUSED(name) _unused_ ## name #endif diff --git a/Include/pymem.h b/Include/pymem.h index e882645757bfd3..a80da99e1dd7fc 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -1,12 +1,8 @@ -/* The PyMem_ family: low-level memory allocation interfaces. - See objimpl.h for the PyObject_ memory family. -*/ +// The PyMem_ family: low-level memory allocation interfaces. +// See objimpl.h for the PyObject_ memory family. #ifndef Py_PYMEM_H #define Py_PYMEM_H - -#include "pyport.h" - #ifdef __cplusplus extern "C" { #endif @@ -91,6 +87,17 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #define PyMem_DEL(p) PyMem_Free((p)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +// Memory allocator which doesn't require the GIL to be held. +// Usually, it's just a thin wrapper to functions of the standard C library: +// malloc(), calloc(), realloc() and free(). The difference is that +// tracemalloc can track these memory allocations. +PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); +PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); +PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); +PyAPI_FUNC(void) PyMem_RawFree(void *ptr); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYMEM_H # include "cpython/pymem.h" @@ -100,5 +107,4 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #ifdef __cplusplus } #endif - -#endif /* !Py_PYMEM_H */ +#endif // !Py_PYMEM_H diff --git a/Include/pyport.h b/Include/pyport.h index d7c6ae64f2bf2f..328471085f959d 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -1,13 +1,8 @@ #ifndef Py_PYPORT_H #define Py_PYPORT_H -#include "pyconfig.h" /* include for defines */ - -#include - -#include #ifndef UCHAR_MAX -# error "limits.h must define UCHAR_MAX" +# error " header must define UCHAR_MAX" #endif #if UCHAR_MAX != 255 # error "Python's source code assumes C's unsigned char is an 8-bit type" @@ -24,9 +19,10 @@ #define _Py_CAST(type, expr) ((type)(expr)) // Static inline functions should use _Py_NULL rather than using directly NULL -// to prevent C++ compiler warnings. On C++11 and newer, _Py_NULL is defined as -// nullptr. -#if defined(__cplusplus) && __cplusplus >= 201103 +// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, +// _Py_NULL is defined as nullptr. +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) # define _Py_NULL nullptr #else # define _Py_NULL NULL @@ -188,68 +184,6 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_MEMCPY memcpy #endif -#ifdef HAVE_IEEEFP_H -#include /* needed for 'finite' declaration on some platforms */ -#endif - -#include /* Moved here from the math section, before extern "C" */ - -/******************************************** - * WRAPPER FOR and/or * - ********************************************/ - -#ifdef HAVE_SYS_TIME_H -#include -#endif -#include - -/****************************** - * WRAPPER FOR * - ******************************/ - -/* NB caller must include */ - -#ifdef HAVE_SYS_SELECT_H -#include -#endif /* !HAVE_SYS_SELECT_H */ - -/******************************* - * stat() and fstat() fiddling * - *******************************/ - -#ifdef HAVE_SYS_STAT_H -#include -#elif defined(HAVE_STAT_H) -#include -#endif - -#ifndef S_IFMT -/* VisualAge C/C++ Failed to Define MountType Field in sys/stat.h */ -#define S_IFMT 0170000 -#endif - -#ifndef S_IFLNK -/* Windows doesn't define S_IFLNK but posixmodule.c maps - * IO_REPARSE_TAG_SYMLINK to S_IFLNK */ -# define S_IFLNK 0120000 -#endif - -#ifndef S_ISREG -#define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - -#ifndef S_ISDIR -#define S_ISDIR(x) (((x) & S_IFMT) == S_IFDIR) -#endif - -#ifndef S_ISCHR -#define S_ISCHR(x) (((x) & S_IFMT) == S_IFCHR) -#endif - -#ifndef S_ISLNK -#define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) -#endif - #ifdef __cplusplus /* Move this down here since some C++ #include's don't like to be included inside an extern "C" */ @@ -421,141 +355,15 @@ extern "C" { # define Py_NO_INLINE #endif -/************************************************************************** -Prototypes that are missing from the standard include files on some systems -(and possibly only some versions of such systems.) - -Please be conservative with adding new ones, document them and enclose them -in platform-specific #ifdefs. -**************************************************************************/ - -#ifdef SOLARIS -/* Unchecked */ -extern int gethostname(char *, int); -#endif - -#ifdef HAVE__GETPTY -#include /* we need to import mode_t */ -extern char * _getpty(int *, int, mode_t, int); -#endif - -/* On QNX 6, struct termio must be declared by including sys/termio.h - if TCGETA, TCSETA, TCSETAW, or TCSETAF are used. sys/termio.h must - be included before termios.h or it will generate an error. */ -#if defined(HAVE_SYS_TERMIO_H) && !defined(__hpux) -#include -#endif - - -/* On 4.4BSD-descendants, ctype functions serves the whole range of - * wchar_t character set rather than single byte code points only. - * This characteristic can break some operations of string object - * including str.upper() and str.split() on UTF-8 locales. This - * workaround was provided by Tim Robbins of FreeBSD project. - */ - -#if defined(__APPLE__) -# define _PY_PORT_CTYPE_UTF8_ISSUE -#endif - -#ifdef _PY_PORT_CTYPE_UTF8_ISSUE -#ifndef __cplusplus - /* The workaround below is unsafe in C++ because - * the defines these symbols as real functions, - * with a slightly different signature. - * See issue #10910 - */ -#include -#include -#undef isalnum -#define isalnum(c) iswalnum(btowc(c)) -#undef isalpha -#define isalpha(c) iswalpha(btowc(c)) -#undef islower -#define islower(c) iswlower(btowc(c)) -#undef isspace -#define isspace(c) iswspace(btowc(c)) -#undef isupper -#define isupper(c) iswupper(btowc(c)) -#undef tolower -#define tolower(c) towlower(btowc(c)) -#undef toupper -#define toupper(c) towupper(btowc(c)) -#endif -#endif - - -/* Declarations for symbol visibility. - - PyAPI_FUNC(type): Declares a public Python API function and return type - PyAPI_DATA(type): Declares public Python data and its type - PyMODINIT_FUNC: A Python module init function. If these functions are - inside the Python core, they are private to the core. - If in an extension module, it may be declared with - external linkage depending on the platform. - - As a number of platforms support/require "__declspec(dllimport/dllexport)", - we support a HAVE_DECLSPEC_DLL macro to save duplication. -*/ - -/* - All windows ports, except cygwin, are handled in PC/pyconfig.h. - - Cygwin is the only other autoconf platform requiring special - linkage handling and it uses __declspec(). -*/ -#if defined(__CYGWIN__) -# define HAVE_DECLSPEC_DLL -#endif - #include "exports.h" -/* only get special linkage if built as shared or platform is Cygwin */ -#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) -# if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE - /* module init functions inside the core need no external linkage */ - /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* -# endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ - /* Building an extension module, or an embedded situation */ - /* public Python functions and data are imported */ - /* Under Cygwin, auto-import functions to prevent compilation */ - /* failures similar to those described at the bottom of 4.1: */ - /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ -# if !defined(__CYGWIN__) -# define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE -# endif /* !__CYGWIN__ */ -# define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE - /* module init functions outside the core must be exported */ -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# endif /* __cplusplus */ -# endif /* Py_BUILD_CORE */ -# endif /* HAVE_DECLSPEC_DLL */ -#endif /* Py_ENABLE_SHARED */ - -/* If no external linkage macros defined by now, create defaults */ -#ifndef PyAPI_FUNC -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -#endif -#ifndef PyAPI_DATA -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE -#endif -#ifndef PyMODINIT_FUNC -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# endif /* __cplusplus */ +#ifdef Py_LIMITED_API + // The internal C API must not be used with the limited C API: make sure + // that Py_BUILD_CORE macro is not defined in this case. These 3 macros are + // used by exports.h, so only undefine them afterwards. +# undef Py_BUILD_CORE +# undef Py_BUILD_CORE_BUILTIN +# undef Py_BUILD_CORE_MODULE #endif /* limits.h constants that may be missing */ @@ -662,6 +470,14 @@ extern char * _getpty(int *, int, mode_t, int); # define WITH_THREAD #endif +/* Some WebAssembly platforms do not provide a working pthread implementation. + * Thread support is stubbed and any attempt to create a new thread fails. + */ +#if (!defined(HAVE_PTHREAD_STUBS) && \ + (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) +# define Py_CAN_START_THREADS 1 +#endif + #ifdef WITH_THREAD # ifdef Py_BUILD_CORE # ifdef HAVE_THREAD_LOCAL @@ -683,12 +499,6 @@ extern char * _getpty(int *, int, mode_t, int); # endif #endif -/* Check that ALT_SOABI is consistent with Py_TRACE_REFS: - ./configure --with-trace-refs should must be used to define Py_TRACE_REFS */ -#if defined(ALT_SOABI) && defined(Py_TRACE_REFS) -# error "Py_TRACE_REFS ABI is not compatible with release and debug ABI" -#endif - #if defined(__ANDROID__) || defined(__VXWORKS__) // Use UTF-8 as the locale encoding, ignore the LC_CTYPE locale. // See _Py_GetLocaleEncoding(), PyUnicode_DecodeLocale() @@ -776,4 +586,16 @@ extern char * _getpty(int *, int, mode_t, int); # define ALIGNOF_MAX_ALIGN_T _Alignof(long double) #endif +#ifndef PY_CXX_CONST +# ifdef __cplusplus +# define PY_CXX_CONST const +# else +# define PY_CXX_CONST +# endif +#endif + +#if defined(__sgi) && !defined(_SGI_MP_SOURCE) +# define _SGI_MP_SOURCE +#endif + #endif /* Py_PYPORT_H */ diff --git a/Include/pystate.h b/Include/pystate.h index e6b4de979c87b8..727b8fbfffe0e6 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -56,7 +56,7 @@ PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *); The caller must hold the GIL. - See also _PyThreadState_UncheckedGet() and _PyThreadState_GET(). */ + See also PyThreadState_GetUnchecked() and _PyThreadState_GET(). */ PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); // Alias to PyThreadState_Get() diff --git a/Include/pystats.h b/Include/pystats.h index 4b961bad2a43e4..acfa32201711e0 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -1,4 +1,9 @@ - +// Statistics on Python performance (public API). +// +// Define _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() +// and Py_DECREF(). +// +// See Include/cpython/pystats.h for the full API. #ifndef Py_PYSTATS_H #define Py_PYSTATS_H @@ -6,105 +11,16 @@ extern "C" { #endif -#ifdef Py_STATS - -#define SPECIALIZATION_FAILURE_KINDS 36 - -/* Stats for determining who is calling PyEval_EvalFrame */ -#define EVAL_CALL_TOTAL 0 -#define EVAL_CALL_VECTOR 1 -#define EVAL_CALL_GENERATOR 2 -#define EVAL_CALL_LEGACY 3 -#define EVAL_CALL_FUNCTION_VECTORCALL 4 -#define EVAL_CALL_BUILD_CLASS 5 -#define EVAL_CALL_SLOT 6 -#define EVAL_CALL_FUNCTION_EX 7 -#define EVAL_CALL_API 8 -#define EVAL_CALL_METHOD 9 - -#define EVAL_CALL_KINDS 10 - -typedef struct _specialization_stats { - uint64_t success; - uint64_t failure; - uint64_t hit; - uint64_t deferred; - uint64_t miss; - uint64_t deopt; - uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS]; -} SpecializationStats; - -typedef struct _opcode_stats { - SpecializationStats specialization; - uint64_t execution_count; - uint64_t pair_count[256]; -} OpcodeStats; - -typedef struct _call_stats { - uint64_t inlined_py_calls; - uint64_t pyeval_calls; - uint64_t frames_pushed; - uint64_t frame_objects_created; - uint64_t eval_calls[EVAL_CALL_KINDS]; -} CallStats; - -typedef struct _object_stats { - uint64_t increfs; - uint64_t decrefs; - uint64_t interpreter_increfs; - uint64_t interpreter_decrefs; - uint64_t allocations; - uint64_t allocations512; - uint64_t allocations4k; - uint64_t allocations_big; - uint64_t frees; - uint64_t to_freelist; - uint64_t from_freelist; - uint64_t new_values; - uint64_t dict_materialized_on_request; - uint64_t dict_materialized_new_key; - uint64_t dict_materialized_too_big; - uint64_t dict_materialized_str_subclass; - uint64_t type_cache_hits; - uint64_t type_cache_misses; - uint64_t type_cache_dunder_hits; - uint64_t type_cache_dunder_misses; - uint64_t type_cache_collisions; -} ObjectStats; - -typedef struct _stats { - OpcodeStats opcode_stats[256]; - CallStats call_stats; - ObjectStats object_stats; -} PyStats; - - -PyAPI_DATA(PyStats) _py_stats_struct; -PyAPI_DATA(PyStats *) _py_stats; - -extern void _Py_StatsClear(void); -extern void _Py_PrintSpecializationStats(int to_file); - -#ifdef _PY_INTERPRETER - -#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_increfs++; } while (0) -#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_decrefs++; } while (0) - -#else - -#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.increfs++; } while (0) -#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.decrefs++; } while (0) - -#endif - +#if defined(Py_STATS) && !defined(Py_LIMITED_API) +# define Py_CPYTHON_PYSTATS_H +# include "cpython/pystats.h" +# undef Py_CPYTHON_PYSTATS_H #else - -#define _Py_INCREF_STAT_INC() ((void)0) -#define _Py_DECREF_STAT_INC() ((void)0) - +# define _Py_INCREF_STAT_INC() ((void)0) +# define _Py_DECREF_STAT_INC() ((void)0) #endif // !Py_STATS #ifdef __cplusplus } #endif -#endif /* !Py_PYSTATs_H */ +#endif // !Py_PYSTATS_H diff --git a/Include/pystrtod.h b/Include/pystrtod.h index fa056d17b6395f..e83d245eb623af 100644 --- a/Include/pystrtod.h +++ b/Include/pystrtod.h @@ -18,15 +18,6 @@ PyAPI_FUNC(char *) PyOS_double_to_string(double val, int flags, int *type); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _Py_string_to_number_with_underscores( - const char *str, Py_ssize_t len, const char *what, PyObject *obj, void *arg, - PyObject *(*innerfunc)(const char *, Py_ssize_t, void *)); - -PyAPI_FUNC(double) _Py_parse_inf_or_nan(const char *p, char **endptr); -#endif - - /* PyOS_double_to_string's "flags" parameter can be set to 0 or more of: */ #define Py_DTSF_SIGN 0x01 /* always add the sign */ #define Py_DTSF_ADD_DOT_0 0x02 /* if the result is an integer add ".0" */ diff --git a/Include/pythread.h b/Include/pythread.h index 63714437c496b7..a3216c51d66165 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -21,7 +21,8 @@ PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); #if (defined(__APPLE__) || defined(__linux__) || defined(_WIN32) \ - || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) \ + || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \ + || defined(__OpenBSD__) || defined(__NetBSD__) \ || defined(__DragonFly__) || defined(_AIX)) #define PY_HAVE_THREAD_NATIVE_ID PyAPI_FUNC(unsigned long) PyThread_get_thread_native_id(void); @@ -33,42 +34,18 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); #define WAIT_LOCK 1 #define NOWAIT_LOCK 0 -/* PY_TIMEOUT_T is the integral type used to specify timeouts when waiting - on a lock (see PyThread_acquire_lock_timed() below). - PY_TIMEOUT_MAX is the highest usable value (in microseconds) of that - type, and depends on the system threading API. - - NOTE: this isn't the same value as `_thread.TIMEOUT_MAX`. The _thread - module exposes a higher-level API, with timeouts expressed in seconds - and floating-point numbers allowed. -*/ +// PY_TIMEOUT_T is the integral type used to specify timeouts when waiting +// on a lock (see PyThread_acquire_lock_timed() below). #define PY_TIMEOUT_T long long -#if defined(_POSIX_THREADS) - /* PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), - convert microseconds to nanoseconds. */ -# define PY_TIMEOUT_MAX (LLONG_MAX / 1000) -#elif defined (NT_THREADS) - // WaitForSingleObject() accepts timeout in milliseconds in the range - // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no - // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. -# if 0xFFFFFFFELL * 1000 < LLONG_MAX -# define PY_TIMEOUT_MAX (0xFFFFFFFELL * 1000) -# else -# define PY_TIMEOUT_MAX LLONG_MAX -# endif -#else -# define PY_TIMEOUT_MAX LLONG_MAX -#endif - /* If microseconds == 0, the call is non-blocking: it returns immediately even when the lock can't be acquired. If microseconds > 0, the call waits up to the specified duration. If microseconds < 0, the call waits until success (or abnormal failure) - microseconds must be less than PY_TIMEOUT_MAX. Behaviour otherwise is - undefined. + If *microseconds* is greater than PY_TIMEOUT_MAX, clamp the timeout to + PY_TIMEOUT_MAX microseconds. If intr_flag is true and the acquire is interrupted by a signal, then the call will return PY_LOCK_INTR. The caller may reattempt to acquire the diff --git a/Include/structseq.h b/Include/structseq.h index 96871155611958..29e24fee54e613 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -31,18 +31,15 @@ PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc); PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type); +PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); +PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); + #ifndef Py_LIMITED_API typedef PyTupleObject PyStructSequence; - -/* Macro, *only* to be used to fill in brand new objects */ -#define PyStructSequence_SET_ITEM(op, i, v) PyTuple_SET_ITEM((op), (i), (v)) - -#define PyStructSequence_GET_ITEM(op, i) PyTuple_GET_ITEM((op), (i)) +#define PyStructSequence_SET_ITEM PyStructSequence_SetItem +#define PyStructSequence_GET_ITEM PyStructSequence_GetItem #endif -PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); -PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); - #ifdef __cplusplus } #endif diff --git a/Include/sysmodule.h b/Include/sysmodule.h index b5087119b1cae7..7b14f72ee2e494 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -1,6 +1,3 @@ - -/* System module interface */ - #ifndef Py_SYSMODULE_H #define Py_SYSMODULE_H #ifdef __cplusplus @@ -10,10 +7,6 @@ extern "C" { PyAPI_FUNC(PyObject *) PySys_GetObject(const char *); PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *); -Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **); -Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int); -Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetPath(const wchar_t *); - PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) @@ -21,14 +14,21 @@ PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); -PyAPI_FUNC(void) PySys_ResetWarnOptions(void); -Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_AddWarnOption(const wchar_t *); -Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_AddWarnOptionUnicode(PyObject *); -Py_DEPRECATED(3.11) PyAPI_FUNC(int) PySys_HasWarnOptions(void); +Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); -Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_AddXOption(const wchar_t *); PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PySys_Audit( + const char *event, + const char *argFormat, + ...); + +PyAPI_FUNC(int) PySys_AuditTuple( + const char *event, + PyObject *args); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_SYSMODULE_H # include "cpython/sysmodule.h" diff --git a/Include/tracemalloc.h b/Include/tracemalloc.h deleted file mode 100644 index bd14217c199c3c..00000000000000 --- a/Include/tracemalloc.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef Py_TRACEMALLOC_H -#define Py_TRACEMALLOC_H - -#ifndef Py_LIMITED_API -/* Track an allocated memory block in the tracemalloc module. - Return 0 on success, return -1 on error (failed to allocate memory to store - the trace). - - Return -2 if tracemalloc is disabled. - - If memory block is already tracked, update the existing trace. */ -PyAPI_FUNC(int) PyTraceMalloc_Track( - unsigned int domain, - uintptr_t ptr, - size_t size); - -/* Untrack an allocated memory block in the tracemalloc module. - Do nothing if the block was not tracked. - - Return -2 if tracemalloc is disabled, otherwise return 0. */ -PyAPI_FUNC(int) PyTraceMalloc_Untrack( - unsigned int domain, - uintptr_t ptr); - -/* Get the traceback where a memory block was allocated. - - Return a tuple of (filename: str, lineno: int) tuples. - - Return None if the tracemalloc module is disabled or if the memory block - is not tracked by tracemalloc. - - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( - unsigned int domain, - uintptr_t ptr); -#endif - -#endif /* !Py_TRACEMALLOC_H */ diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 5839c747a29275..dee00715b3c51d 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1,8 +1,6 @@ #ifndef Py_UNICODEOBJECT_H #define Py_UNICODEOBJECT_H -#include // va_list - /* Unicode implementation based on original code by Fredrik Lundh, @@ -55,8 +53,6 @@ Copyright (c) Corporation for National Research Initiatives. * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * -------------------------------------------------------------------- */ -#include - /* === Internal API ======================================================= */ /* --- Internal Unicode Format -------------------------------------------- */ @@ -93,10 +89,6 @@ Copyright (c) Corporation for National Research Initiatives. # endif #endif -#ifdef HAVE_WCHAR_H -# include -#endif - /* Py_UCS4 and Py_UCS2 are typedefs for the respective unicode representations. */ typedef uint32_t Py_UCS4; @@ -965,6 +957,15 @@ PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString( const char *right /* ASCII-encoded string */ ); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 +/* Compare a Unicode object with UTF-8 encoded C string. + Return 1 if they are equal, or 0 otherwise. + This function does not raise exceptions. */ + +PyAPI_FUNC(int) PyUnicode_EqualToUTF8(PyObject *, const char *); +PyAPI_FUNC(int) PyUnicode_EqualToUTF8AndSize(PyObject *, const char *, Py_ssize_t); +#endif + /* Rich compare two strings and return one of the following: - NULL in case an exception was raised diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index 8e1fa1b9286a77..727ba6934bbacb 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -27,7 +27,8 @@ PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, PyObject *callback); PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, PyObject *callback); -PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); +Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); +PyAPI_FUNC(int) PyWeakref_GetRef(PyObject *ref, PyObject **pobj); #ifndef Py_LIMITED_API diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 65a94b6b1bdfd5..439f8c02f4b586 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -22,7 +22,6 @@ 'tkMessageBox': 'tkinter.messagebox', 'ScrolledText': 'tkinter.scrolledtext', 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', 'ttk': 'tkinter.ttk', 'Tkinter': 'tkinter', 'markupbase': '_markupbase', diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py new file mode 100644 index 00000000000000..5dd06ae487dfcf --- /dev/null +++ b/Lib/_opcode_metadata.py @@ -0,0 +1,335 @@ +# This file is generated by Tools/cases_generator/generate_cases.py +# from: +# Python/bytecodes.c +# Do not edit! + +_specializations = { + "RESUME": [ + "RESUME_CHECK", + ], + "TO_BOOL": [ + "TO_BOOL_ALWAYS_TRUE", + "TO_BOOL_BOOL", + "TO_BOOL_INT", + "TO_BOOL_LIST", + "TO_BOOL_NONE", + "TO_BOOL_STR", + ], + "BINARY_OP": [ + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + ], + "BINARY_SUBSCR": [ + "BINARY_SUBSCR_DICT", + "BINARY_SUBSCR_GETITEM", + "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_STR_INT", + "BINARY_SUBSCR_TUPLE_INT", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ], + "SEND": [ + "SEND_GEN", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ], + "STORE_ATTR": [ + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ], + "LOAD_SUPER_ATTR": [ + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + ], + "COMPARE_OP": [ + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ], + "FOR_ITER": [ + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + ], + "CALL": [ + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_PY_WITH_DEFAULTS", + "CALL_TYPE_1", + "CALL_STR_1", + "CALL_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_O", + "CALL_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_LEN", + "CALL_ISINSTANCE", + "CALL_LIST_APPEND", + "CALL_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_NOARGS", + "CALL_METHOD_DESCRIPTOR_FAST", + "CALL_ALLOC_AND_ENTER_INIT", + ], +} + +# An irregular case: +_specializations["BINARY_OP"].append("BINARY_OP_INPLACE_ADD_UNICODE") + +_specialized_opmap = { + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, + 'BINARY_OP_ADD_FLOAT': 150, + 'BINARY_OP_ADD_INT': 151, + 'BINARY_OP_ADD_UNICODE': 152, + 'BINARY_OP_MULTIPLY_FLOAT': 153, + 'BINARY_OP_MULTIPLY_INT': 154, + 'BINARY_OP_SUBTRACT_FLOAT': 155, + 'BINARY_OP_SUBTRACT_INT': 156, + 'BINARY_SUBSCR_DICT': 157, + 'BINARY_SUBSCR_GETITEM': 158, + 'BINARY_SUBSCR_LIST_INT': 159, + 'BINARY_SUBSCR_STR_INT': 160, + 'BINARY_SUBSCR_TUPLE_INT': 161, + 'CALL_ALLOC_AND_ENTER_INIT': 162, + 'CALL_BOUND_METHOD_EXACT_ARGS': 163, + 'CALL_BUILTIN_CLASS': 164, + 'CALL_BUILTIN_FAST': 165, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 166, + 'CALL_BUILTIN_O': 167, + 'CALL_ISINSTANCE': 168, + 'CALL_LEN': 169, + 'CALL_LIST_APPEND': 170, + 'CALL_METHOD_DESCRIPTOR_FAST': 171, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 172, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 173, + 'CALL_METHOD_DESCRIPTOR_O': 174, + 'CALL_PY_EXACT_ARGS': 175, + 'CALL_PY_WITH_DEFAULTS': 176, + 'CALL_STR_1': 177, + 'CALL_TUPLE_1': 178, + 'CALL_TYPE_1': 179, + 'COMPARE_OP_FLOAT': 180, + 'COMPARE_OP_INT': 181, + 'COMPARE_OP_STR': 182, + 'FOR_ITER_GEN': 183, + 'FOR_ITER_LIST': 184, + 'FOR_ITER_RANGE': 185, + 'FOR_ITER_TUPLE': 186, + 'LOAD_ATTR_CLASS': 187, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 188, + 'LOAD_ATTR_INSTANCE_VALUE': 189, + 'LOAD_ATTR_METHOD_LAZY_DICT': 190, + 'LOAD_ATTR_METHOD_NO_DICT': 191, + 'LOAD_ATTR_METHOD_WITH_VALUES': 192, + 'LOAD_ATTR_MODULE': 193, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 194, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 195, + 'LOAD_ATTR_PROPERTY': 196, + 'LOAD_ATTR_SLOT': 197, + 'LOAD_ATTR_WITH_HINT': 198, + 'LOAD_GLOBAL_BUILTIN': 199, + 'LOAD_GLOBAL_MODULE': 200, + 'LOAD_SUPER_ATTR_ATTR': 201, + 'LOAD_SUPER_ATTR_METHOD': 202, + 'RESUME_CHECK': 203, + 'SEND_GEN': 204, + 'STORE_ATTR_INSTANCE_VALUE': 205, + 'STORE_ATTR_SLOT': 206, + 'STORE_ATTR_WITH_HINT': 207, + 'STORE_SUBSCR_DICT': 208, + 'STORE_SUBSCR_LIST_INT': 209, + 'TO_BOOL_ALWAYS_TRUE': 210, + 'TO_BOOL_BOOL': 211, + 'TO_BOOL_INT': 212, + 'TO_BOOL_LIST': 213, + 'TO_BOOL_NONE': 214, + 'TO_BOOL_STR': 215, + 'UNPACK_SEQUENCE_LIST': 216, + 'UNPACK_SEQUENCE_TUPLE': 217, + 'UNPACK_SEQUENCE_TWO_TUPLE': 218, +} + +opmap = { + 'CACHE': 0, + 'BEFORE_ASYNC_WITH': 1, + 'BEFORE_WITH': 2, + 'BINARY_SLICE': 4, + 'BINARY_SUBSCR': 5, + 'CHECK_EG_MATCH': 6, + 'CHECK_EXC_MATCH': 7, + 'CLEANUP_THROW': 8, + 'DELETE_SUBSCR': 9, + 'END_ASYNC_FOR': 10, + 'END_FOR': 11, + 'END_SEND': 12, + 'EXIT_INIT_CHECK': 13, + 'FORMAT_SIMPLE': 14, + 'FORMAT_WITH_SPEC': 15, + 'GET_AITER': 16, + 'RESERVED': 17, + 'GET_ANEXT': 18, + 'GET_ITER': 19, + 'GET_LEN': 20, + 'GET_YIELD_FROM_ITER': 21, + 'INTERPRETER_EXIT': 22, + 'LOAD_ASSERTION_ERROR': 23, + 'LOAD_BUILD_CLASS': 24, + 'LOAD_LOCALS': 25, + 'MAKE_FUNCTION': 26, + 'MATCH_KEYS': 27, + 'MATCH_MAPPING': 28, + 'MATCH_SEQUENCE': 29, + 'NOP': 30, + 'POP_EXCEPT': 31, + 'POP_TOP': 32, + 'PUSH_EXC_INFO': 33, + 'PUSH_NULL': 34, + 'RETURN_GENERATOR': 35, + 'RETURN_VALUE': 36, + 'SETUP_ANNOTATIONS': 37, + 'STORE_SLICE': 38, + 'STORE_SUBSCR': 39, + 'TO_BOOL': 40, + 'UNARY_INVERT': 41, + 'UNARY_NEGATIVE': 42, + 'UNARY_NOT': 43, + 'WITH_EXCEPT_START': 44, + 'BINARY_OP': 45, + 'BUILD_CONST_KEY_MAP': 46, + 'BUILD_LIST': 47, + 'BUILD_MAP': 48, + 'BUILD_SET': 49, + 'BUILD_SLICE': 50, + 'BUILD_STRING': 51, + 'BUILD_TUPLE': 52, + 'CALL': 53, + 'CALL_FUNCTION_EX': 54, + 'CALL_INTRINSIC_1': 55, + 'CALL_INTRINSIC_2': 56, + 'CALL_KW': 57, + 'COMPARE_OP': 58, + 'CONTAINS_OP': 59, + 'CONVERT_VALUE': 60, + 'COPY': 61, + 'COPY_FREE_VARS': 62, + 'DELETE_ATTR': 63, + 'DELETE_DEREF': 64, + 'DELETE_FAST': 65, + 'DELETE_GLOBAL': 66, + 'DELETE_NAME': 67, + 'DICT_MERGE': 68, + 'DICT_UPDATE': 69, + 'ENTER_EXECUTOR': 70, + 'EXTENDED_ARG': 71, + 'FOR_ITER': 72, + 'GET_AWAITABLE': 73, + 'IMPORT_FROM': 74, + 'IMPORT_NAME': 75, + 'IS_OP': 76, + 'JUMP_BACKWARD': 77, + 'JUMP_BACKWARD_NO_INTERRUPT': 78, + 'JUMP_FORWARD': 79, + 'LIST_APPEND': 80, + 'LIST_EXTEND': 81, + 'LOAD_ATTR': 82, + 'LOAD_CONST': 83, + 'LOAD_DEREF': 84, + 'LOAD_FAST': 85, + 'LOAD_FAST_AND_CLEAR': 86, + 'LOAD_FAST_CHECK': 87, + 'LOAD_FAST_LOAD_FAST': 88, + 'LOAD_FROM_DICT_OR_DEREF': 89, + 'LOAD_FROM_DICT_OR_GLOBALS': 90, + 'LOAD_GLOBAL': 91, + 'LOAD_NAME': 92, + 'LOAD_SUPER_ATTR': 93, + 'MAKE_CELL': 94, + 'MAP_ADD': 95, + 'MATCH_CLASS': 96, + 'POP_JUMP_IF_FALSE': 97, + 'POP_JUMP_IF_NONE': 98, + 'POP_JUMP_IF_NOT_NONE': 99, + 'POP_JUMP_IF_TRUE': 100, + 'RAISE_VARARGS': 101, + 'RERAISE': 102, + 'RETURN_CONST': 103, + 'SEND': 104, + 'SET_ADD': 105, + 'SET_FUNCTION_ATTRIBUTE': 106, + 'SET_UPDATE': 107, + 'STORE_ATTR': 108, + 'STORE_DEREF': 109, + 'STORE_FAST': 110, + 'STORE_FAST_LOAD_FAST': 111, + 'STORE_FAST_STORE_FAST': 112, + 'STORE_GLOBAL': 113, + 'STORE_NAME': 114, + 'SWAP': 115, + 'UNPACK_EX': 116, + 'UNPACK_SEQUENCE': 117, + 'YIELD_VALUE': 118, + 'RESUME': 149, + 'INSTRUMENTED_RESUME': 236, + 'INSTRUMENTED_END_FOR': 237, + 'INSTRUMENTED_END_SEND': 238, + 'INSTRUMENTED_RETURN_VALUE': 239, + 'INSTRUMENTED_RETURN_CONST': 240, + 'INSTRUMENTED_YIELD_VALUE': 241, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 242, + 'INSTRUMENTED_FOR_ITER': 243, + 'INSTRUMENTED_CALL': 244, + 'INSTRUMENTED_CALL_KW': 245, + 'INSTRUMENTED_CALL_FUNCTION_EX': 246, + 'INSTRUMENTED_INSTRUCTION': 247, + 'INSTRUMENTED_JUMP_FORWARD': 248, + 'INSTRUMENTED_JUMP_BACKWARD': 249, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'INSTRUMENTED_LINE': 254, + 'JUMP': 256, + 'JUMP_NO_INTERRUPT': 257, + 'LOAD_CLOSURE': 258, + 'LOAD_METHOD': 259, + 'LOAD_SUPER_METHOD': 260, + 'LOAD_ZERO_SUPER_ATTR': 261, + 'LOAD_ZERO_SUPER_METHOD': 262, + 'POP_BLOCK': 263, + 'SETUP_CLEANUP': 264, + 'SETUP_FINALLY': 265, + 'SETUP_WITH': 266, + 'STORE_FAST_MAYBE_NULL': 267, +} +MIN_INSTRUMENTED_OPCODE = 236 +HAVE_ARGUMENT = 45 diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index f4fc2c58e5e293..bca2acf1fc88cf 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1015,13 +1015,9 @@ def fromisocalendar(cls, year, week, day): def __repr__(self): """Convert to formal string, for repr(). - >>> dt = datetime(2010, 1, 1) - >>> repr(dt) - 'datetime.datetime(2010, 1, 1, 0, 0)' - - >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) - >>> repr(dt) - 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' + >>> d = date(2010, 1, 1) + >>> repr(d) + 'datetime.date(2010, 1, 1)' """ return "%s.%s(%d, %d, %d)" % (_get_class_module(self), self.__class__.__qualname__, @@ -1112,6 +1108,8 @@ def replace(self, year=None, month=None, day=None): day = self._day return type(self)(year, month, day) + __replace__ = replace + # Comparisons of date objects with other. def __eq__(self, other): @@ -1236,7 +1234,7 @@ def __reduce__(self): class tzinfo: """Abstract base class for time zone info classes. - Subclasses must override the name(), utcoffset() and dst() methods. + Subclasses must override the tzname(), utcoffset() and dst() methods. """ __slots__ = () @@ -1637,6 +1635,8 @@ def replace(self, hour=None, minute=None, second=None, microsecond=None, fold = self._fold return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold) + __replace__ = replace + # Pickle support. def _getstate(self, protocol=3): @@ -1684,7 +1684,7 @@ class datetime(date): The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. """ - __slots__ = date.__slots__ + time.__slots__ + __slots__ = time.__slots__ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0): @@ -1812,7 +1812,7 @@ def utcfromtimestamp(cls, t): warnings.warn("datetime.utcfromtimestamp() is deprecated and scheduled " "for removal in a future version. Use timezone-aware " "objects to represent datetimes in UTC: " - "datetime.fromtimestamp(t, datetime.UTC).", + "datetime.datetime.fromtimestamp(t, datetime.UTC).", DeprecationWarning, stacklevel=2) return cls._fromtimestamp(t, True, None) @@ -1830,7 +1830,7 @@ def utcnow(cls): warnings.warn("datetime.utcnow() is deprecated and scheduled for " "removal in a future version. Instead, Use timezone-aware " "objects to represent datetimes in UTC: " - "datetime.now(datetime.UTC).", + "datetime.datetime.now(datetime.UTC).", DeprecationWarning, stacklevel=2) t = _time.time() @@ -1983,6 +1983,8 @@ def replace(self, year=None, month=None, day=None, hour=None, return type(self)(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold) + __replace__ = replace + def _local_timezone(self): if self.tzinfo is None: ts = self._mktime() diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 7f247ff47c9e61..32698abac78d25 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -33,11 +33,8 @@ # Rebind for compatibility BlockingIOError = BlockingIOError -# Does io.IOBase finalizer log the exception if the close() method fails? -# The exception is ignored silently by default in release build. -_IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode) # Does open() check its 'errors' argument? -_CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE +_CHECK_ERRORS = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode) def text_encoding(encoding, stacklevel=2): @@ -416,18 +413,9 @@ def __del__(self): if closed: return - if _IOBASE_EMITS_UNRAISABLE: - self.close() - else: - # The try/except block is in case this is called at program - # exit time, when it's possible that globals have already been - # deleted, and then the close() call might fail. Since - # there's nothing we can do about such failures and they annoy - # the end users, we suppress the traceback. - try: - self.close() - except: - pass + # If close() fails, the caller logs the exception with + # sys.unraisablehook. close() must be called at the end at __del__(). + self.close() ### Inquiries ### diff --git a/Lib/aifc.py b/Lib/aifc.py deleted file mode 100644 index 5254987e22bc16..00000000000000 --- a/Lib/aifc.py +++ /dev/null @@ -1,984 +0,0 @@ -"""Stuff to parse AIFF-C and AIFF files. - -Unless explicitly stated otherwise, the description below is true -both for AIFF-C files and AIFF files. - -An AIFF-C file has the following structure. - - +-----------------+ - | FORM | - +-----------------+ - | | - +----+------------+ - | | AIFC | - | +------------+ - | | | - | | . | - | | . | - | | . | - +----+------------+ - -An AIFF file has the string "AIFF" instead of "AIFC". - -A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, -big endian order), followed by the data. The size field does not include -the size of the 8 byte header. - -The following chunk types are recognized. - - FVER - (AIFF-C only). - MARK - <# of markers> (2 bytes) - list of markers: - (2 bytes, must be > 0) - (4 bytes) - ("pstring") - COMM - <# of channels> (2 bytes) - <# of sound frames> (4 bytes) - (2 bytes) - (10 bytes, IEEE 80-bit extended - floating point) - in AIFF-C files only: - (4 bytes) - ("pstring") - SSND - (4 bytes, not used by this program) - (4 bytes, not used by this program) - - -A pstring consists of 1 byte length, a string of characters, and 0 or 1 -byte pad to make the total length even. - -Usage. - -Reading AIFF files: - f = aifc.open(file, 'r') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods read(), seek(), and close(). -In some types of audio files, if the setpos() method is not used, -the seek() method is not necessary. - -This returns an instance of a class with the following public methods: - getnchannels() -- returns number of audio channels (1 for - mono, 2 for stereo) - getsampwidth() -- returns sample width in bytes - getframerate() -- returns sampling frequency - getnframes() -- returns number of audio frames - getcomptype() -- returns compression type ('NONE' for AIFF files) - getcompname() -- returns human-readable version of - compression type ('not compressed' for AIFF files) - getparams() -- returns a namedtuple consisting of all of the - above in the above order - getmarkers() -- get the list of marks in the audio file or None - if there are no marks - getmark(id) -- get mark with the specified id (raises an error - if the mark does not exist) - readframes(n) -- returns at most n frames of audio - rewind() -- rewind to the beginning of the audio stream - setpos(pos) -- seek to the specified position - tell() -- return the current position - close() -- close the instance (make it unusable) -The position returned by tell(), the position given to setpos() and -the position of marks are all compatible and have nothing to do with -the actual position in the file. -The close() method is called automatically when the class instance -is destroyed. - -Writing AIFF files: - f = aifc.open(file, 'w') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods write(), tell(), seek(), and -close(). - -This returns an instance of a class with the following public methods: - aiff() -- create an AIFF file (AIFF-C default) - aifc() -- create an AIFF-C file - setnchannels(n) -- set the number of channels - setsampwidth(n) -- set the sample width - setframerate(n) -- set the frame rate - setnframes(n) -- set the number of frames - setcomptype(type, name) - -- set the compression type and the - human-readable compression type - setparams(tuple) - -- set all parameters at once - setmark(id, pos, name) - -- add specified mark to the list of marks - tell() -- return current position in output file (useful - in combination with setmark()) - writeframesraw(data) - -- write audio frames without pathing up the - file header - writeframes(data) - -- write audio frames and patch up the file header - close() -- patch up the file header and close the - output file -You should set the parameters before the first writeframesraw or -writeframes. The total number of frames does not need to be set, -but when it is set to the correct value, the header does not have to -be patched up. -It is best to first set all parameters, perhaps possibly the -compression type, and then write audio frames using writeframesraw. -When all frames have been written, either call writeframes(b'') or -close() to patch up the sizes in the header. -Marks can be added anytime. If there are any marks, you must call -close() after all frames have been written. -The close() method is called automatically when the class instance -is destroyed. - -When a file is opened with the extension '.aiff', an AIFF file is -written, otherwise an AIFF-C file is written. This default can be -changed by calling aiff() or aifc() before the first writeframes or -writeframesraw. -""" - -import struct -import builtins -import warnings - -__all__ = ["Error", "open"] - - -warnings._deprecated(__name__, remove=(3, 13)) - - -class Error(Exception): - pass - -_AIFC_version = 0xA2805140 # Version 1 of AIFF-C - -def _read_long(file): - try: - return struct.unpack('>l', file.read(4))[0] - except struct.error: - raise EOFError from None - -def _read_ulong(file): - try: - return struct.unpack('>L', file.read(4))[0] - except struct.error: - raise EOFError from None - -def _read_short(file): - try: - return struct.unpack('>h', file.read(2))[0] - except struct.error: - raise EOFError from None - -def _read_ushort(file): - try: - return struct.unpack('>H', file.read(2))[0] - except struct.error: - raise EOFError from None - -def _read_string(file): - length = ord(file.read(1)) - if length == 0: - data = b'' - else: - data = file.read(length) - if length & 1 == 0: - dummy = file.read(1) - return data - -_HUGE_VAL = 1.79769313486231e+308 # See - -def _read_float(f): # 10 bytes - expon = _read_short(f) # 2 bytes - sign = 1 - if expon < 0: - sign = -1 - expon = expon + 0x8000 - himant = _read_ulong(f) # 4 bytes - lomant = _read_ulong(f) # 4 bytes - if expon == himant == lomant == 0: - f = 0.0 - elif expon == 0x7FFF: - f = _HUGE_VAL - else: - expon = expon - 16383 - f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63) - return sign * f - -def _write_short(f, x): - f.write(struct.pack('>h', x)) - -def _write_ushort(f, x): - f.write(struct.pack('>H', x)) - -def _write_long(f, x): - f.write(struct.pack('>l', x)) - -def _write_ulong(f, x): - f.write(struct.pack('>L', x)) - -def _write_string(f, s): - if len(s) > 255: - raise ValueError("string exceeds maximum pstring length") - f.write(struct.pack('B', len(s))) - f.write(s) - if len(s) & 1 == 0: - f.write(b'\x00') - -def _write_float(f, x): - import math - if x < 0: - sign = 0x8000 - x = x * -1 - else: - sign = 0 - if x == 0: - expon = 0 - himant = 0 - lomant = 0 - else: - fmant, expon = math.frexp(x) - if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN - expon = sign|0x7FFF - himant = 0 - lomant = 0 - else: # Finite - expon = expon + 16382 - if expon < 0: # denormalized - fmant = math.ldexp(fmant, expon) - expon = 0 - expon = expon | sign - fmant = math.ldexp(fmant, 32) - fsmant = math.floor(fmant) - himant = int(fsmant) - fmant = math.ldexp(fmant - fsmant, 32) - fsmant = math.floor(fmant) - lomant = int(fsmant) - _write_ushort(f, expon) - _write_ulong(f, himant) - _write_ulong(f, lomant) - -with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - from chunk import Chunk -from collections import namedtuple - -_aifc_params = namedtuple('_aifc_params', - 'nchannels sampwidth framerate nframes comptype compname') - -_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' -_aifc_params.sampwidth.__doc__ = 'Sample width in bytes' -_aifc_params.framerate.__doc__ = 'Sampling frequency' -_aifc_params.nframes.__doc__ = 'Number of audio frames' -_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' -_aifc_params.compname.__doc__ = ("""\ -A human-readable version of the compression type -('not compressed' for AIFF files)""") - - -class Aifc_read: - # Variables used in this class: - # - # These variables are available to the user though appropriate - # methods of this class: - # _file -- the open file with methods read(), close(), and seek() - # set through the __init__() method - # _nchannels -- the number of audio channels - # available through the getnchannels() method - # _nframes -- the number of audio frames - # available through the getnframes() method - # _sampwidth -- the number of bytes per audio sample - # available through the getsampwidth() method - # _framerate -- the sampling frequency - # available through the getframerate() method - # _comptype -- the AIFF-C compression type ('NONE' if AIFF) - # available through the getcomptype() method - # _compname -- the human-readable AIFF-C compression type - # available through the getcomptype() method - # _markers -- the marks in the audio file - # available through the getmarkers() and getmark() - # methods - # _soundpos -- the position in the audio stream - # available through the tell() method, set through the - # setpos() method - # - # These variables are used internally only: - # _version -- the AIFF-C version number - # _decomp -- the decompressor from builtin module cl - # _comm_chunk_read -- 1 iff the COMM chunk has been read - # _aifc -- 1 iff reading an AIFF-C file - # _ssnd_seek_needed -- 1 iff positioned correctly in audio - # file for readframes() - # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk - # _framesize -- size of one frame in the file - - _file = None # Set here since __del__ checks it - - def initfp(self, file): - self._version = 0 - self._convert = None - self._markers = [] - self._soundpos = 0 - self._file = file - chunk = Chunk(file) - if chunk.getname() != b'FORM': - raise Error('file does not start with FORM id') - formdata = chunk.read(4) - if formdata == b'AIFF': - self._aifc = 0 - elif formdata == b'AIFC': - self._aifc = 1 - else: - raise Error('not an AIFF or AIFF-C file') - self._comm_chunk_read = 0 - self._ssnd_chunk = None - while 1: - self._ssnd_seek_needed = 1 - try: - chunk = Chunk(self._file) - except EOFError: - break - chunkname = chunk.getname() - if chunkname == b'COMM': - self._read_comm_chunk(chunk) - self._comm_chunk_read = 1 - elif chunkname == b'SSND': - self._ssnd_chunk = chunk - dummy = chunk.read(8) - self._ssnd_seek_needed = 0 - elif chunkname == b'FVER': - self._version = _read_ulong(chunk) - elif chunkname == b'MARK': - self._readmark(chunk) - chunk.skip() - if not self._comm_chunk_read or not self._ssnd_chunk: - raise Error('COMM chunk and/or SSND chunk missing') - - def __init__(self, f): - if isinstance(f, str): - file_object = builtins.open(f, 'rb') - try: - self.initfp(file_object) - except: - file_object.close() - raise - else: - # assume it is an open file object already - self.initfp(f) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - # - # User visible methods. - # - def getfp(self): - return self._file - - def rewind(self): - self._ssnd_seek_needed = 1 - self._soundpos = 0 - - def close(self): - file = self._file - if file is not None: - self._file = None - file.close() - - def tell(self): - return self._soundpos - - def getnchannels(self): - return self._nchannels - - def getnframes(self): - return self._nframes - - def getsampwidth(self): - return self._sampwidth - - def getframerate(self): - return self._framerate - - def getcomptype(self): - return self._comptype - - def getcompname(self): - return self._compname - -## def getversion(self): -## return self._version - - def getparams(self): - return _aifc_params(self.getnchannels(), self.getsampwidth(), - self.getframerate(), self.getnframes(), - self.getcomptype(), self.getcompname()) - - def getmarkers(self): - if len(self._markers) == 0: - return None - return self._markers - - def getmark(self, id): - for marker in self._markers: - if id == marker[0]: - return marker - raise Error('marker {0!r} does not exist'.format(id)) - - def setpos(self, pos): - if pos < 0 or pos > self._nframes: - raise Error('position not in range') - self._soundpos = pos - self._ssnd_seek_needed = 1 - - def readframes(self, nframes): - if self._ssnd_seek_needed: - self._ssnd_chunk.seek(0) - dummy = self._ssnd_chunk.read(8) - pos = self._soundpos * self._framesize - if pos: - self._ssnd_chunk.seek(pos + 8) - self._ssnd_seek_needed = 0 - if nframes == 0: - return b'' - data = self._ssnd_chunk.read(nframes * self._framesize) - if self._convert and data: - data = self._convert(data) - self._soundpos = self._soundpos + len(data) // (self._nchannels - * self._sampwidth) - return data - - # - # Internal methods. - # - - def _alaw2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.alaw2lin(data, 2) - - def _ulaw2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.ulaw2lin(data, 2) - - def _adpcm2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - if not hasattr(self, '_adpcmstate'): - # first time - self._adpcmstate = None - data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate) - return data - - def _sowt2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.byteswap(data, 2) - - def _read_comm_chunk(self, chunk): - self._nchannels = _read_short(chunk) - self._nframes = _read_long(chunk) - self._sampwidth = (_read_short(chunk) + 7) // 8 - self._framerate = int(_read_float(chunk)) - if self._sampwidth <= 0: - raise Error('bad sample width') - if self._nchannels <= 0: - raise Error('bad # of channels') - self._framesize = self._nchannels * self._sampwidth - if self._aifc: - #DEBUG: SGI's soundeditor produces a bad size :-( - kludge = 0 - if chunk.chunksize == 18: - kludge = 1 - warnings.warn('Warning: bad COMM chunk size') - chunk.chunksize = 23 - #DEBUG end - self._comptype = chunk.read(4) - #DEBUG start - if kludge: - length = ord(chunk.file.read(1)) - if length & 1 == 0: - length = length + 1 - chunk.chunksize = chunk.chunksize + length - chunk.file.seek(-1, 1) - #DEBUG end - self._compname = _read_string(chunk) - if self._comptype != b'NONE': - if self._comptype == b'G722': - self._convert = self._adpcm2lin - elif self._comptype in (b'ulaw', b'ULAW'): - self._convert = self._ulaw2lin - elif self._comptype in (b'alaw', b'ALAW'): - self._convert = self._alaw2lin - elif self._comptype in (b'sowt', b'SOWT'): - self._convert = self._sowt2lin - else: - raise Error('unsupported compression type') - self._sampwidth = 2 - else: - self._comptype = b'NONE' - self._compname = b'not compressed' - - def _readmark(self, chunk): - nmarkers = _read_short(chunk) - # Some files appear to contain invalid counts. - # Cope with this by testing for EOF. - try: - for i in range(nmarkers): - id = _read_short(chunk) - pos = _read_long(chunk) - name = _read_string(chunk) - if pos or name: - # some files appear to have - # dummy markers consisting of - # a position 0 and name '' - self._markers.append((id, pos, name)) - except EOFError: - w = ('Warning: MARK chunk contains only %s marker%s instead of %s' % - (len(self._markers), '' if len(self._markers) == 1 else 's', - nmarkers)) - warnings.warn(w) - -class Aifc_write: - # Variables used in this class: - # - # These variables are user settable through appropriate methods - # of this class: - # _file -- the open file with methods write(), close(), tell(), seek() - # set through the __init__() method - # _comptype -- the AIFF-C compression type ('NONE' in AIFF) - # set through the setcomptype() or setparams() method - # _compname -- the human-readable AIFF-C compression type - # set through the setcomptype() or setparams() method - # _nchannels -- the number of audio channels - # set through the setnchannels() or setparams() method - # _sampwidth -- the number of bytes per audio sample - # set through the setsampwidth() or setparams() method - # _framerate -- the sampling frequency - # set through the setframerate() or setparams() method - # _nframes -- the number of audio frames written to the header - # set through the setnframes() or setparams() method - # _aifc -- whether we're writing an AIFF-C file or an AIFF file - # set through the aifc() method, reset through the - # aiff() method - # - # These variables are used internally only: - # _version -- the AIFF-C version number - # _comp -- the compressor from builtin module cl - # _nframeswritten -- the number of audio frames actually written - # _datalength -- the size of the audio samples written to the header - # _datawritten -- the size of the audio samples actually written - - _file = None # Set here since __del__ checks it - - def __init__(self, f): - if isinstance(f, str): - file_object = builtins.open(f, 'wb') - try: - self.initfp(file_object) - except: - file_object.close() - raise - - # treat .aiff file extensions as non-compressed audio - if f.endswith('.aiff'): - self._aifc = 0 - else: - # assume it is an open file object already - self.initfp(f) - - def initfp(self, file): - self._file = file - self._version = _AIFC_version - self._comptype = b'NONE' - self._compname = b'not compressed' - self._convert = None - self._nchannels = 0 - self._sampwidth = 0 - self._framerate = 0 - self._nframes = 0 - self._nframeswritten = 0 - self._datawritten = 0 - self._datalength = 0 - self._markers = [] - self._marklength = 0 - self._aifc = 1 # AIFF-C is default - - def __del__(self): - self.close() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - # - # User visible methods. - # - def aiff(self): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._aifc = 0 - - def aifc(self): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._aifc = 1 - - def setnchannels(self, nchannels): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if nchannels < 1: - raise Error('bad # of channels') - self._nchannels = nchannels - - def getnchannels(self): - if not self._nchannels: - raise Error('number of channels not set') - return self._nchannels - - def setsampwidth(self, sampwidth): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if sampwidth < 1 or sampwidth > 4: - raise Error('bad sample width') - self._sampwidth = sampwidth - - def getsampwidth(self): - if not self._sampwidth: - raise Error('sample width not set') - return self._sampwidth - - def setframerate(self, framerate): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if framerate <= 0: - raise Error('bad frame rate') - self._framerate = framerate - - def getframerate(self): - if not self._framerate: - raise Error('frame rate not set') - return self._framerate - - def setnframes(self, nframes): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._nframes = nframes - - def getnframes(self): - return self._nframeswritten - - def setcomptype(self, comptype, compname): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if comptype not in (b'NONE', b'ulaw', b'ULAW', - b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'): - raise Error('unsupported compression type') - self._comptype = comptype - self._compname = compname - - def getcomptype(self): - return self._comptype - - def getcompname(self): - return self._compname - -## def setversion(self, version): -## if self._nframeswritten: -## raise Error, 'cannot change parameters after starting to write' -## self._version = version - - def setparams(self, params): - nchannels, sampwidth, framerate, nframes, comptype, compname = params - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if comptype not in (b'NONE', b'ulaw', b'ULAW', - b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'): - raise Error('unsupported compression type') - self.setnchannels(nchannels) - self.setsampwidth(sampwidth) - self.setframerate(framerate) - self.setnframes(nframes) - self.setcomptype(comptype, compname) - - def getparams(self): - if not self._nchannels or not self._sampwidth or not self._framerate: - raise Error('not all parameters set') - return _aifc_params(self._nchannels, self._sampwidth, self._framerate, - self._nframes, self._comptype, self._compname) - - def setmark(self, id, pos, name): - if id <= 0: - raise Error('marker ID must be > 0') - if pos < 0: - raise Error('marker position must be >= 0') - if not isinstance(name, bytes): - raise Error('marker name must be bytes') - for i in range(len(self._markers)): - if id == self._markers[i][0]: - self._markers[i] = id, pos, name - return - self._markers.append((id, pos, name)) - - def getmark(self, id): - for marker in self._markers: - if id == marker[0]: - return marker - raise Error('marker {0!r} does not exist'.format(id)) - - def getmarkers(self): - if len(self._markers) == 0: - return None - return self._markers - - def tell(self): - return self._nframeswritten - - def writeframesraw(self, data): - if not isinstance(data, (bytes, bytearray)): - data = memoryview(data).cast('B') - self._ensure_header_written(len(data)) - nframes = len(data) // (self._sampwidth * self._nchannels) - if self._convert: - data = self._convert(data) - self._file.write(data) - self._nframeswritten = self._nframeswritten + nframes - self._datawritten = self._datawritten + len(data) - - def writeframes(self, data): - self.writeframesraw(data) - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - - def close(self): - if self._file is None: - return - try: - self._ensure_header_written(0) - if self._datawritten & 1: - # quick pad to even size - self._file.write(b'\x00') - self._datawritten = self._datawritten + 1 - self._writemarkers() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten or \ - self._marklength: - self._patchheader() - finally: - # Prevent ref cycles - self._convert = None - f = self._file - self._file = None - f.close() - - # - # Internal methods. - # - - def _lin2alaw(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.lin2alaw(data, 2) - - def _lin2ulaw(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.lin2ulaw(data, 2) - - def _lin2adpcm(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - if not hasattr(self, '_adpcmstate'): - self._adpcmstate = None - data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate) - return data - - def _lin2sowt(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.byteswap(data, 2) - - def _ensure_header_written(self, datasize): - if not self._nframeswritten: - if self._comptype in (b'ULAW', b'ulaw', - b'ALAW', b'alaw', b'G722', - b'sowt', b'SOWT'): - if not self._sampwidth: - self._sampwidth = 2 - if self._sampwidth != 2: - raise Error('sample width must be 2 when compressing ' - 'with ulaw/ULAW, alaw/ALAW, sowt/SOWT ' - 'or G7.22 (ADPCM)') - if not self._nchannels: - raise Error('# channels not specified') - if not self._sampwidth: - raise Error('sample width not specified') - if not self._framerate: - raise Error('sampling rate not specified') - self._write_header(datasize) - - def _init_compression(self): - if self._comptype == b'G722': - self._convert = self._lin2adpcm - elif self._comptype in (b'ulaw', b'ULAW'): - self._convert = self._lin2ulaw - elif self._comptype in (b'alaw', b'ALAW'): - self._convert = self._lin2alaw - elif self._comptype in (b'sowt', b'SOWT'): - self._convert = self._lin2sowt - - def _write_header(self, initlength): - if self._aifc and self._comptype != b'NONE': - self._init_compression() - self._file.write(b'FORM') - if not self._nframes: - self._nframes = initlength // (self._nchannels * self._sampwidth) - self._datalength = self._nframes * self._nchannels * self._sampwidth - if self._datalength & 1: - self._datalength = self._datalength + 1 - if self._aifc: - if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'): - self._datalength = self._datalength // 2 - if self._datalength & 1: - self._datalength = self._datalength + 1 - elif self._comptype == b'G722': - self._datalength = (self._datalength + 3) // 4 - if self._datalength & 1: - self._datalength = self._datalength + 1 - try: - self._form_length_pos = self._file.tell() - except (AttributeError, OSError): - self._form_length_pos = None - commlength = self._write_form_length(self._datalength) - if self._aifc: - self._file.write(b'AIFC') - self._file.write(b'FVER') - _write_ulong(self._file, 4) - _write_ulong(self._file, self._version) - else: - self._file.write(b'AIFF') - self._file.write(b'COMM') - _write_ulong(self._file, commlength) - _write_short(self._file, self._nchannels) - if self._form_length_pos is not None: - self._nframes_pos = self._file.tell() - _write_ulong(self._file, self._nframes) - if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'): - _write_short(self._file, 8) - else: - _write_short(self._file, self._sampwidth * 8) - _write_float(self._file, self._framerate) - if self._aifc: - self._file.write(self._comptype) - _write_string(self._file, self._compname) - self._file.write(b'SSND') - if self._form_length_pos is not None: - self._ssnd_length_pos = self._file.tell() - _write_ulong(self._file, self._datalength + 8) - _write_ulong(self._file, 0) - _write_ulong(self._file, 0) - - def _write_form_length(self, datalength): - if self._aifc: - commlength = 18 + 5 + len(self._compname) - if commlength & 1: - commlength = commlength + 1 - verslength = 12 - else: - commlength = 18 - verslength = 0 - _write_ulong(self._file, 4 + verslength + self._marklength + \ - 8 + commlength + 16 + datalength) - return commlength - - def _patchheader(self): - curpos = self._file.tell() - if self._datawritten & 1: - datalength = self._datawritten + 1 - self._file.write(b'\x00') - else: - datalength = self._datawritten - if datalength == self._datalength and \ - self._nframes == self._nframeswritten and \ - self._marklength == 0: - self._file.seek(curpos, 0) - return - self._file.seek(self._form_length_pos, 0) - dummy = self._write_form_length(datalength) - self._file.seek(self._nframes_pos, 0) - _write_ulong(self._file, self._nframeswritten) - self._file.seek(self._ssnd_length_pos, 0) - _write_ulong(self._file, datalength + 8) - self._file.seek(curpos, 0) - self._nframes = self._nframeswritten - self._datalength = datalength - - def _writemarkers(self): - if len(self._markers) == 0: - return - self._file.write(b'MARK') - length = 2 - for marker in self._markers: - id, pos, name = marker - length = length + len(name) + 1 + 6 - if len(name) & 1 == 0: - length = length + 1 - _write_ulong(self._file, length) - self._marklength = length + 8 - _write_short(self._file, len(self._markers)) - for marker in self._markers: - id, pos, name = marker - _write_short(self._file, id) - _write_ulong(self._file, pos) - _write_string(self._file, name) - -def open(f, mode=None): - if mode is None: - if hasattr(f, 'mode'): - mode = f.mode - else: - mode = 'rb' - if mode in ('r', 'rb'): - return Aifc_read(f) - elif mode in ('w', 'wb'): - return Aifc_write(f) - else: - raise Error("mode must be 'r', 'rb', 'w', or 'wb'") - - -if __name__ == '__main__': - import sys - if not sys.argv[1:]: - sys.argv.append('/usr/demos/data/audio/bach.aiff') - fn = sys.argv[1] - with open(fn, 'r') as f: - print("Reading", fn) - print("nchannels =", f.getnchannels()) - print("nframes =", f.getnframes()) - print("sampwidth =", f.getsampwidth()) - print("framerate =", f.getframerate()) - print("comptype =", f.getcomptype()) - print("compname =", f.getcompname()) - if sys.argv[2:]: - gn = sys.argv[2] - print("Writing", gn) - with open(gn, 'w') as g: - g.setparams(f.getparams()) - while 1: - data = f.readframes(1024) - if not data: - break - g.writeframes(data) - print("Done.") diff --git a/Lib/argparse.py b/Lib/argparse.py index f5f44ff02c0d38..a32884db80d1ea 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -89,8 +89,6 @@ import re as _re import sys as _sys -import warnings - from gettext import gettext as _, ngettext SUPPRESS = '==SUPPRESS==' @@ -883,16 +881,19 @@ def __call__(self, parser, namespace, values, option_string=None): raise NotImplementedError(_('.__call__() not defined')) +# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. +_deprecated_default = object() + class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=None, - choices=None, + type=_deprecated_default, + choices=_deprecated_default, required=False, help=None, - metavar=None): + metavar=_deprecated_default): _option_strings = [] for option_string in option_strings: @@ -902,6 +903,25 @@ def __init__(self, option_string = '--no-' + option_string[2:] _option_strings.append(option_string) + # We need `_deprecated` special value to ban explicit arguments that + # match default value. Like: + # parser.add_argument('-f', action=BooleanOptionalAction, type=int) + for field_name in ('type', 'choices', 'metavar'): + if locals()[field_name] is not _deprecated_default: + import warnings + warnings._deprecated( + field_name, + "{name!r} is deprecated as of Python 3.12 and will be " + "removed in Python {remove}.", + remove=(3, 14)) + + if type is _deprecated_default: + type = None + if choices is _deprecated_default: + choices = None + if metavar is _deprecated_default: + metavar = None + super().__init__( option_strings=_option_strings, dest=dest, @@ -1507,6 +1527,8 @@ def _add_container_actions(self, container): title_group_map = {} for group in self._action_groups: if group.title in title_group_map: + # This branch could happen if a derived class added + # groups with duplicated titles in __init__ msg = _('cannot merge actions - two groups are named %r') raise ValueError(msg % (group.title)) title_group_map[group.title] = group @@ -1677,6 +1699,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_argument_group(self, *args, **kwargs): + import warnings warnings.warn( "Nesting argument groups is deprecated.", category=DeprecationWarning, @@ -1705,6 +1728,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_mutually_exclusive_group(self, *args, **kwargs): + import warnings warnings.warn( "Nesting mutually exclusive groups is deprecated.", category=DeprecationWarning, @@ -1790,13 +1814,11 @@ def identity(string): # add parent arguments and defaults for parent in parents: + if not isinstance(parent, ArgumentParser): + raise TypeError('parents must be a list of ArgumentParser') self._add_container_actions(parent) - try: - defaults = parent._defaults - except AttributeError: - pass - else: - self._defaults.update(defaults) + defaults = parent._defaults + self._defaults.update(defaults) # ======================= # Pretty __repr__ methods diff --git a/Lib/ast.py b/Lib/ast.py index 65152047a22370..f7888d18859ae4 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -32,13 +32,15 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None): + type_comments=False, feature_version=None, optimize=-1): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). Pass type_comments=True to get back type comments where the syntax allows. """ flags = PyCF_ONLY_AST + if optimize > 0: + flags |= PyCF_OPTIMIZED_AST if type_comments: flags |= PyCF_TYPE_COMMENTS if feature_version is None: @@ -50,7 +52,7 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version) + _feature_version=feature_version, optimize=optimize) def literal_eval(node_or_string): @@ -1051,6 +1053,8 @@ def visit_ClassDef(self, node): self.fill("@") self.traverse(deco) self.fill("class " + node.name) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) with self.delimit_if("(", ")", condition = node.bases or node.keywords): comma = False for e in node.bases: @@ -1082,6 +1086,8 @@ def _function_helper(self, node, fill_suffix): self.traverse(deco) def_str = fill_suffix + " " + node.name self.fill(def_str) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) with self.delimit("(", ")"): self.traverse(node.args) if node.returns: @@ -1090,6 +1096,30 @@ def _function_helper(self, node, fill_suffix): with self.block(extra=self.get_type_comment(node)): self._write_docstring_and_traverse_body(node) + def _type_params_helper(self, type_params): + if type_params is not None and len(type_params) > 0: + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, type_params) + + def visit_TypeVar(self, node): + self.write(node.name) + if node.bound: + self.write(": ") + self.traverse(node.bound) + + def visit_TypeVarTuple(self, node): + self.write("*" + node.name) + + def visit_ParamSpec(self, node): + self.write("**" + node.name) + + def visit_TypeAlias(self, node): + self.fill("type ") + self.traverse(node.name) + self._type_params_helper(node.type_params) + self.write(" = ") + self.traverse(node.value) + def visit_For(self, node): self._for_helper("for ", node) @@ -1195,17 +1225,7 @@ def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): def visit_JoinedStr(self, node): self.write("f") - if self._avoid_backslashes: - with self.buffered() as buffer: - self._write_fstring_inner(node) - return self._write_str_avoiding_backslashes("".join(buffer)) - - # If we don't need to avoid backslashes globally (i.e., we only need - # to avoid them inside FormattedValues), it's cosmetically preferred - # to use escaped whitespace. That is, it's preferred to use backslashes - # for cases like: f"{x}\n". To accomplish this, we keep track of what - # in our buffer corresponds to FormattedValues and what corresponds to - # Constant parts of the f-string, and allow escapes accordingly. + fstring_parts = [] for value in node.values: with self.buffered() as buffer: @@ -1216,25 +1236,49 @@ def visit_JoinedStr(self, node): new_fstring_parts = [] quote_types = list(_ALL_QUOTES) + fallback_to_repr = False for value, is_constant in fstring_parts: - value, quote_types = self._str_literal_helper( - value, - quote_types=quote_types, - escape_special_whitespace=is_constant, - ) + if is_constant: + value, new_quote_types = self._str_literal_helper( + value, + quote_types=quote_types, + escape_special_whitespace=True, + ) + if set(new_quote_types).isdisjoint(quote_types): + fallback_to_repr = True + break + quote_types = new_quote_types + elif "\n" in value: + quote_types = [q for q in quote_types if q in _MULTI_QUOTES] + assert quote_types new_fstring_parts.append(value) + if fallback_to_repr: + # If we weren't able to find a quote type that works for all parts + # of the JoinedStr, fallback to using repr and triple single quotes. + quote_types = ["'''"] + new_fstring_parts.clear() + for value, is_constant in fstring_parts: + if is_constant: + value = repr('"' + value) # force repr to use single quotes + expected_prefix = "'\"" + assert value.startswith(expected_prefix), repr(value) + value = value[len(expected_prefix):-1] + new_fstring_parts.append(value) + value = "".join(new_fstring_parts) quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") - def _write_fstring_inner(self, node): + def _write_fstring_inner(self, node, scape_newlines=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: - self._write_fstring_inner(value) + self._write_fstring_inner(value, scape_newlines=scape_newlines) elif isinstance(node, Constant) and isinstance(node.value, str): value = node.value.replace("{", "{{").replace("}", "}}") + if scape_newlines: + value = value.replace("\n", "\\n") self.write(value) elif isinstance(node, FormattedValue): self.visit_FormattedValue(node) @@ -1243,16 +1287,12 @@ def _write_fstring_inner(self, node): def visit_FormattedValue(self, node): def unparse_inner(inner): - unparser = type(self)(_avoid_backslashes=True) + unparser = type(self)() unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) with self.delimit("{", "}"): expr = unparse_inner(node.value) - if "\\" in expr: - raise ValueError( - "Unable to avoid backslash in f-string expression part" - ) if expr.startswith("{"): # Separate pair of opening brackets as "{ {" self.write(" ") @@ -1261,7 +1301,10 @@ def unparse_inner(inner): self.write(f"!{chr(node.conversion)}") if node.format_spec: self.write(":") - self._write_fstring_inner(node.format_spec) + self._write_fstring_inner( + node.format_spec, + scape_newlines=True + ) def visit_Name(self, node): self.write(node.id) diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index fed16ec7c67fac..03165a425eb7d2 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -34,6 +34,7 @@ streams.__all__ + subprocess.__all__ + tasks.__all__ + + taskgroups.__all__ + threads.__all__ + timeouts.__all__ + transports.__all__) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 32d7e1c481ecc5..416c732298d9a9 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -305,7 +305,7 @@ def _wakeup(self): self._waiters = None for waiter in waiters: if not waiter.done(): - waiter.set_result(waiter) + waiter.set_result(None) def _start_serving(self): if self._serving: @@ -377,7 +377,27 @@ async def serve_forever(self): self._serving_forever_fut = None async def wait_closed(self): - if self._waiters is None or self._active_count == 0: + """Wait until server is closed and all connections are dropped. + + - If the server is not closed, wait. + - If it is closed, but there are still active connections, wait. + + Anyone waiting here will be unblocked once both conditions + (server is closed and all connections have been dropped) + have become true, in either order. + + Historical note: In 3.11 and before, this was broken, returning + immediately if the server was already closed, even if there + were still active connections. An attempted fix in 3.12.0 was + still broken, returning immediately if the server was still + open and there were no active connections. Hopefully in 3.12.1 + we have it right. + """ + # Waiters are unblocked by self._wakeup(), which is called + # from two places: self.close() and self._detach(), but only + # when both conditions have become true. To signal that this + # has happened, self._wakeup() sets self._waiters to None. + if self._waiters is None: return waiter = self._loop.create_future() self._waiters.append(waiter) @@ -400,6 +420,8 @@ def __init__(self): self._clock_resolution = time.get_clock_info('monotonic').resolution self._exception_handler = None self.set_debug(coroutines._is_debug_mode()) + # The preserved state of async generator hooks. + self._old_agen_hooks = None # In debug mode, if the execution of a callback or a step of a task # exceed this duration in seconds, the slow callback/task is logged. self.slow_callback_duration = 0.1 @@ -443,7 +465,7 @@ def create_task(self, coro, *, name=None, context=None): else: task = self._task_factory(self, coro, context=context) - tasks._set_task_name(task, name) + task.set_name(name) return task @@ -601,29 +623,52 @@ def _check_running(self): raise RuntimeError( 'Cannot run the event loop while another loop is running') - def run_forever(self): - """Run until stop() is called.""" + def _run_forever_setup(self): + """Prepare the run loop to process events. + + This method exists so that custom custom event loop subclasses (e.g., event loops + that integrate a GUI event loop with Python's event loop) have access to all the + loop setup logic. + """ self._check_closed() self._check_running() self._set_coroutine_origin_tracking(self._debug) - old_agen_hooks = sys.get_asyncgen_hooks() - try: - self._thread_id = threading.get_ident() - sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, - finalizer=self._asyncgen_finalizer_hook) + self._old_agen_hooks = sys.get_asyncgen_hooks() + self._thread_id = threading.get_ident() + sys.set_asyncgen_hooks( + firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook + ) - events._set_running_loop(self) + events._set_running_loop(self) + + def _run_forever_cleanup(self): + """Clean up after an event loop finishes the looping over events. + + This method exists so that custom custom event loop subclasses (e.g., event loops + that integrate a GUI event loop with Python's event loop) have access to all the + loop cleanup logic. + """ + self._stopping = False + self._thread_id = None + events._set_running_loop(None) + self._set_coroutine_origin_tracking(False) + # Restore any pre-existing async generator hooks. + if self._old_agen_hooks is not None: + sys.set_asyncgen_hooks(*self._old_agen_hooks) + self._old_agen_hooks = None + + def run_forever(self): + """Run until stop() is called.""" + try: + self._run_forever_setup() while True: self._run_once() if self._stopping: break finally: - self._stopping = False - self._thread_id = None - events._set_running_loop(None) - self._set_coroutine_origin_tracking(False) - sys.set_asyncgen_hooks(*old_agen_hooks) + self._run_forever_cleanup() def run_until_complete(self, future): """Run until the Future is done. @@ -727,7 +772,7 @@ def call_later(self, delay, callback, *args, context=None): always relative to the current time. Each callback will be called exactly once. If two callbacks - are scheduled for exactly the same time, it undefined which + are scheduled for exactly the same time, it is undefined which will be called first. Any positional arguments after the callback will be passed to @@ -1907,8 +1952,11 @@ def _run_once(self): timeout = 0 elif self._scheduled: # Compute the desired timeout. - when = self._scheduled[0]._when - timeout = min(max(0, when - self.time()), MAXIMUM_SELECT_TIMEOUT) + timeout = self._scheduled[0]._when - self.time() + if timeout > MAXIMUM_SELECT_TIMEOUT: + timeout = MAXIMUM_SELECT_TIMEOUT + elif timeout < 0: + timeout = 0 event_list = self._selector.select(timeout) self._process_events(event_list) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index ce44942186b272..0ccf85105e6673 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -617,7 +617,7 @@ class AbstractEventLoopPolicy: def get_event_loop(self): """Get the event loop for the current context. - Returns an event loop object implementing the BaseEventLoop interface, + Returns an event loop object implementing the AbstractEventLoop interface, or raises an exception in case no event loop has been set for the current context and the current policy does not specify to create one. diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index fa2422b7fba4a7..d521b4e2e255a9 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -274,9 +274,8 @@ def _ensure_fd_no_transport(self, fd): def _add_reader(self, fd, callback, *args): self._check_closed() handle = events.Handle(callback, args, self, None) - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: self._selector.register(fd, selectors.EVENT_READ, (handle, None)) else: @@ -290,30 +289,27 @@ def _add_reader(self, fd, callback, *args): def _remove_reader(self, fd): if self.is_closed(): return False - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: return False + mask, (reader, writer) = key.events, key.data + mask &= ~selectors.EVENT_READ + if not mask: + self._selector.unregister(fd) else: - mask, (reader, writer) = key.events, key.data - mask &= ~selectors.EVENT_READ - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (None, writer)) + self._selector.modify(fd, mask, (None, writer)) - if reader is not None: - reader.cancel() - return True - else: - return False + if reader is not None: + reader.cancel() + return True + else: + return False def _add_writer(self, fd, callback, *args): self._check_closed() handle = events.Handle(callback, args, self, None) - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: self._selector.register(fd, selectors.EVENT_WRITE, (None, handle)) else: @@ -328,24 +324,22 @@ def _remove_writer(self, fd): """Remove a writer callback.""" if self.is_closed(): return False - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: return False + mask, (reader, writer) = key.events, key.data + # Remove both writer and connector. + mask &= ~selectors.EVENT_WRITE + if not mask: + self._selector.unregister(fd) else: - mask, (reader, writer) = key.events, key.data - # Remove both writer and connector. - mask &= ~selectors.EVENT_WRITE - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (reader, None)) + self._selector.modify(fd, mask, (reader, None)) - if writer is not None: - writer.cancel() - return True - else: - return False + if writer is not None: + writer.cancel() + return True + else: + return False def add_reader(self, fd, callback, *args): """Add a reader callback.""" @@ -1202,6 +1196,7 @@ def _reset_empty_waiter(self): def close(self): self._read_ready_cb = None + self._write_ready = None super().close() diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 488e17d8bccd5b..3eb65a8a08b5a0 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -539,7 +539,7 @@ def _start_handshake(self): # start handshake timeout count down self._handshake_timeout_handle = \ self._loop.call_later(self._ssl_handshake_timeout, - lambda: self._check_handshake_timeout()) + self._check_handshake_timeout) self._do_handshake() @@ -619,7 +619,7 @@ def _start_shutdown(self): self._set_state(SSLProtocolState.FLUSHING) self._shutdown_timeout_handle = self._loop.call_later( self._ssl_shutdown_timeout, - lambda: self._check_shutdown_timeout() + self._check_shutdown_timeout ) self._do_flush() @@ -758,7 +758,7 @@ def _do_read__buffered(self): else: break else: - self._loop.call_soon(lambda: self._do_read()) + self._loop.call_soon(self._do_read) except SSLAgainErrors: pass if offset > 0: diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index bf15f517e50dce..ffb48b9cd6cb70 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -5,6 +5,7 @@ import collections import socket import sys +import warnings import weakref if hasattr(socket, 'AF_UNIX'): @@ -66,9 +67,8 @@ async def start_server(client_connected_cb, host=None, port=None, *, positional host and port, with various optional keyword arguments following. The return value is the same as loop.create_server(). - Additional optional keyword arguments are loop (to set the event loop - instance to use) and limit (to set the buffer limit passed to the - StreamReader). + Additional optional keyword argument is limit (to set the buffer + limit passed to the StreamReader). The return value is the same as loop.create_server(), i.e. a Server object which can be used to stop the service. @@ -245,7 +245,19 @@ def connection_made(self, transport): res = self._client_connected_cb(reader, self._stream_writer) if coroutines.iscoroutine(res): + def callback(task): + exc = task.exception() + if exc is not None: + self._loop.call_exception_handler({ + 'message': 'Unhandled exception in client_connected_cb', + 'exception': exc, + 'transport': transport, + }) + transport.close() + self._task = self._loop.create_task(res) + self._task.add_done_callback(callback) + self._strong_reader = None def connection_lost(self, exc): @@ -392,6 +404,13 @@ async def start_tls(self, sslcontext, *, self._transport = new_transport protocol._replace_writer(self) + def __del__(self, warnings=warnings): + if not self._transport.is_closing(): + if self._loop.is_closed(): + warnings.warn("loop is closed", ResourceWarning) + else: + self.close() + warnings.warn(f"unclosed {self!r}", ResourceWarning) class StreamReader: diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index 50727ca300e63e..043359bbd03f8a 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -81,6 +81,9 @@ def pipe_connection_lost(self, fd, exc): self._stdin_closed.set_result(None) else: self._stdin_closed.set_exception(exc) + # Since calling `wait_closed()` is not mandatory, + # we shouldn't log the traceback if this is not awaited. + self._stdin_closed._log_traceback = False return if fd == 1: reader = self.stdout @@ -144,15 +147,17 @@ def kill(self): async def _feed_stdin(self, input): debug = self._loop.get_debug() - if input is not None: - self.stdin.write(input) - if debug: - logger.debug( - '%r communicate: feed stdin (%s bytes)', self, len(input)) try: + if input is not None: + self.stdin.write(input) + if debug: + logger.debug( + '%r communicate: feed stdin (%s bytes)', self, len(input)) + await self.stdin.drain() except (BrokenPipeError, ConnectionResetError) as exc: - # communicate() ignores BrokenPipeError and ConnectionResetError + # communicate() ignores BrokenPipeError and ConnectionResetError. + # write() and drain() can raise these exceptions. if debug: logger.debug('%r communicate: stdin got %r', self, exc) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 06b2e0db86a1fe..91be0decc41c42 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -2,7 +2,7 @@ # license: PSFL. -__all__ = ["TaskGroup"] +__all__ = ("TaskGroup",) from . import events from . import exceptions @@ -54,16 +54,14 @@ def __repr__(self): async def __aenter__(self): if self._entered: raise RuntimeError( - f"TaskGroup {self!r} has been already entered") - self._entered = True - + f"TaskGroup {self!r} has already been entered") if self._loop is None: self._loop = events.get_running_loop() - self._parent_task = tasks.current_task(self._loop) if self._parent_task is None: raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') + self._entered = True return self @@ -163,7 +161,7 @@ def create_task(self, coro, *, name=None, context=None): task = self._loop.create_task(coro) else: task = self._loop.create_task(coro, context=context) - tasks._set_task_name(task, name) + task.set_name(name) # optimization: Immediately call the done callback if the task is # already done (e.g. if the coro was able to complete eagerly), # and skip scheduling a done callback diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 8d5bde09ea9b5b..e84b21390557be 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -15,8 +15,8 @@ import functools import inspect import itertools +import math import types -import warnings import weakref from types import GenericAlias @@ -67,34 +67,31 @@ def all_tasks(loop=None): if futures._get_loop(t) is loop and not t.done()} -def _set_task_name(task, name): - if name is not None: - try: - set_name = task.set_name - except AttributeError: - warnings.warn("Task.set_name() was added in Python 3.8, " - "the method support will be mandatory for third-party " - "task implementations since 3.13.", - DeprecationWarning, stacklevel=3) - else: - set_name(name) - - class Task(futures._PyFuture): # Inherit Python Task implementation # from a Python Future implementation. """A coroutine wrapped in a Future.""" # An important invariant maintained while a Task not done: + # _fut_waiter is either None or a Future. The Future + # can be either done() or not done(). + # The task can be in any of 3 states: # - # - Either _fut_waiter is None, and _step() is scheduled; - # - or _fut_waiter is some Future, and _step() is *not* scheduled. + # - 1: _fut_waiter is not None and not _fut_waiter.done(): + # __step() is *not* scheduled and the Task is waiting for _fut_waiter. + # - 2: (_fut_waiter is None or _fut_waiter.done()) and __step() is scheduled: + # the Task is waiting for __step() to be executed. + # - 3: _fut_waiter is None and __step() is *not* scheduled: + # the Task is currently executing (in __step()). # - # The only transition from the latter to the former is through - # _wakeup(). When _fut_waiter is not None, one of its callbacks - # must be _wakeup(). - - # If False, don't log a message if the task is destroyed whereas its + # * In state 1, one of the callbacks of __fut_waiter must be __wakeup(). + # * The transition from 1 to 2 happens when _fut_waiter becomes done(), + # as it schedules __wakeup() to be called (which calls __step() so + # we way that __step() is scheduled). + # * It transitions from 2 to 3 when __step() is executed, and it clears + # _fut_waiter to None. + + # If False, don't log a message if the task is destroyed while its # status is still pending _log_destroy_pending = True @@ -411,7 +408,7 @@ def create_task(coro, *, name=None, context=None): else: task = loop.create_task(coro, context=context) - _set_task_name(task, name) + task.set_name(name) return task @@ -427,8 +424,6 @@ async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED): The fs iterable must not be empty. - Coroutines will be wrapped in Tasks. - Returns two sets of Future: (done, pending). Usage: @@ -646,6 +641,9 @@ async def sleep(delay, result=None): await __sleep0() return result + if math.isnan(delay): + raise ValueError("Invalid delay: NaN (not a number)") + loop = events.get_running_loop() future = loop.create_future() h = loop.call_later(delay, @@ -944,21 +942,21 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any callable + that returns a Task-compatible object and has a signature compatible + with `Task.__init__`; it must have the `eager_start` keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in + Most applications will use `Task` for `custom_task_constructor` and in this case there's no need to call `create_eager_task_factory()` directly. Instead the global `eager_task_factory` instance can be used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + """ def factory(loop, coro, *, name=None, context=None): return custom_task_constructor( diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 029c468739bf2d..30042abb3ad804 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -49,8 +49,9 @@ def when(self) -> Optional[float]: def reschedule(self, when: Optional[float]) -> None: """Reschedule the timeout.""" - assert self._state is not _State.CREATED if self._state is not _State.ENTERED: + if self._state is _State.CREATED: + raise RuntimeError("Timeout has not been entered") raise RuntimeError( f"Cannot change state of {self._state.value} Timeout", ) @@ -82,11 +83,14 @@ def __repr__(self) -> str: return f"" async def __aenter__(self) -> "Timeout": + if self._state is not _State.CREATED: + raise RuntimeError("Timeout has already been entered") + task = tasks.current_task() + if task is None: + raise RuntimeError("Timeout should be used inside a task") self._state = _State.ENTERED - self._task = tasks.current_task() + self._task = task self._cancelling = self._task.cancelling() - if self._task is None: - raise RuntimeError("Timeout should be used inside a task") self.reschedule(self._when) return self diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 17fb4d5f7646ce..41ccf1b78fb93b 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -32,6 +32,7 @@ 'FastChildWatcher', 'PidfdChildWatcher', 'MultiLoopChildWatcher', 'ThreadedChildWatcher', 'DefaultEventLoopPolicy', + 'EventLoop', ) @@ -63,6 +64,7 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} + self._unix_server_sockets = {} def close(self): super().close() @@ -226,8 +228,7 @@ async def _make_subprocess_transport(self, protocol, args, shell, return transp def _child_watcher_callback(self, pid, returncode, transp): - # Skip one iteration for callbacks to be executed - self.call_soon_threadsafe(self.call_soon, transp._process_exited, returncode) + self.call_soon_threadsafe(transp._process_exited, returncode) async def create_unix_connection( self, protocol_factory, path=None, *, @@ -284,7 +285,7 @@ async def create_unix_server( sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - start_serving=True): + start_serving=True, cleanup_socket=True): if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') @@ -340,6 +341,15 @@ async def create_unix_server( raise ValueError( f'A UNIX Domain Stream Socket was expected, got {sock!r}') + if cleanup_socket: + path = sock.getsockname() + # Check for abstract socket. `str` and `bytes` paths are supported. + if path[0] not in (0, '\x00'): + try: + self._unix_server_sockets[sock] = os.stat(path).st_ino + except FileNotFoundError: + pass + sock.setblocking(False) server = base_events.Server(self, [sock], protocol_factory, ssl, backlog, ssl_handshake_timeout, @@ -394,6 +404,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fut.set_result(total_sent) return + # On 32-bit architectures truncate to 1GiB to avoid OverflowError + blocksize = min(blocksize, sys.maxsize//2 + 1) + try: sent = os.sendfile(fd, fileno, offset, blocksize) except (BlockingIOError, InterruptedError): @@ -457,6 +470,27 @@ def cb(fut): self.remove_writer(fd) fut.add_done_callback(cb) + def _stop_serving(self, sock): + # Is this a unix socket that needs cleanup? + if sock in self._unix_server_sockets: + path = sock.getsockname() + else: + path = None + + super()._stop_serving(sock) + + if path is not None: + prev_ino = self._unix_server_sockets[sock] + del self._unix_server_sockets[sock] + try: + if os.stat(path).st_ino == prev_ino: + os.unlink(path) + except FileNotFoundError: + pass + except OSError as err: + logger.error('Unable to clean up listening UNIX socket ' + '%r: %r', path, err) + class _UnixReadPipeTransport(transports.ReadTransport): @@ -1368,14 +1402,7 @@ def is_active(self): return True def close(self): - self._join_threads() - - def _join_threads(self): - """Internal: Join all non-daemon threads""" - threads = [thread for thread in list(self._threads.values()) - if thread.is_alive() and not thread.daemon] - for thread in threads: - thread.join() + pass def __enter__(self): return self @@ -1394,7 +1421,7 @@ def __del__(self, _warn=warnings.warn): def add_child_handler(self, pid, callback, *args): loop = events.get_running_loop() thread = threading.Thread(target=self._do_waitpid, - name=f"waitpid-{next(self._pid_counter)}", + name=f"asyncio-waitpid-{next(self._pid_counter)}", args=(loop, pid, callback, args), daemon=True) self._threads[pid] = thread @@ -1462,8 +1489,6 @@ def _init_watcher(self): self._watcher = PidfdChildWatcher() else: self._watcher = ThreadedChildWatcher() - if threading.current_thread() is threading.main_thread(): - self._watcher.attach_loop(self._local._loop) def set_event_loop(self, loop): """Set the event loop. @@ -1508,3 +1533,4 @@ def set_child_watcher(self, watcher): SelectorEventLoop = _UnixSelectorEventLoop DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index c9a5fb841cb134..b62ea75fee3858 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -29,7 +29,7 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', 'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy', - 'WindowsProactorEventLoopPolicy', + 'WindowsProactorEventLoopPolicy', 'EventLoop', ) @@ -314,24 +314,25 @@ def __init__(self, proactor=None): proactor = IocpProactor() super().__init__(proactor) - def run_forever(self): - try: - assert self._self_reading_future is None - self.call_soon(self._loop_self_reading) - super().run_forever() - finally: - if self._self_reading_future is not None: - ov = self._self_reading_future._ov - self._self_reading_future.cancel() - # self_reading_future was just cancelled so if it hasn't been - # finished yet, it never will be (it's possible that it has - # already finished and its callback is waiting in the queue, - # where it could still happen if the event loop is restarted). - # Unregister it otherwise IocpProactor.close will wait for it - # forever - if ov is not None: - self._proactor._unregister(ov) - self._self_reading_future = None + def _run_forever_setup(self): + assert self._self_reading_future is None + self.call_soon(self._loop_self_reading) + super()._run_forever_setup() + + def _run_forever_cleanup(self): + super()._run_forever_cleanup() + if self._self_reading_future is not None: + ov = self._self_reading_future._ov + self._self_reading_future.cancel() + # self_reading_future was just cancelled so if it hasn't been + # finished yet, it never will be (it's possible that it has + # already finished and its callback is waiting in the queue, + # where it could still happen if the event loop is restarted). + # Unregister it otherwise IocpProactor.close will wait for it + # forever + if ov is not None: + self._proactor._unregister(ov) + self._self_reading_future = None async def create_pipe_connection(self, protocol_factory, address): f = self._proactor.connect_pipe(address) @@ -894,3 +895,4 @@ class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy +EventLoop = ProactorEventLoop diff --git a/Lib/base64.py b/Lib/base64.py index e233647ee76639..e3e983b3064fe7 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -164,7 +164,6 @@ def urlsafe_b64decode(s): _b32rev = {} def _b32encode(alphabet, s): - global _b32tab2 # Delay the initialization of the table to not waste memory # if the function is never called if alphabet not in _b32tab2: @@ -200,7 +199,6 @@ def _b32encode(alphabet, s): return bytes(encoded) def _b32decode(alphabet, s, casefold=False, map01=None): - global _b32rev # Delay the initialization of the table to not waste memory # if the function is never called if alphabet not in _b32rev: diff --git a/Lib/bdb.py b/Lib/bdb.py index 0f3eec653baaad..1acf7957f0d669 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -32,6 +32,7 @@ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} + self.frame_trace_lines = {} self.frame_returning = None self._load_breaks() @@ -331,6 +332,9 @@ def set_trace(self, frame=None): while frame: frame.f_trace = self.trace_dispatch self.botframe = frame + # We need f_trace_liens == True for the debugger to work + self.frame_trace_lines[frame] = frame.f_trace_lines + frame.f_trace_lines = True frame = frame.f_back self.set_step() sys.settrace(self.trace_dispatch) @@ -349,6 +353,9 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back + for frame, prev_trace_lines in self.frame_trace_lines.items(): + frame.f_trace_lines = prev_trace_lines + self.frame_trace_lines = {} def set_quit(self): """Set quitting attribute to True. diff --git a/Lib/calendar.py b/Lib/calendar.py index ea56f12ccc41d0..03469d8ac96bcd 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -10,7 +10,6 @@ from enum import IntEnum, global_enum import locale as _locale from itertools import repeat -import warnings __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", @@ -44,6 +43,7 @@ def __str__(self): def __getattr__(name): if name in ('January', 'February'): + import warnings warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead", DeprecationWarning, stacklevel=2) if name == 'January': @@ -585,8 +585,6 @@ def __enter__(self): _locale.setlocale(_locale.LC_TIME, self.locale) def __exit__(self, *args): - if self.oldlocale is None: - return _locale.setlocale(_locale.LC_TIME, self.oldlocale) @@ -690,7 +688,7 @@ def timegm(tuple): return seconds -def main(args): +def main(args=None): import argparse parser = argparse.ArgumentParser() textgroup = parser.add_argument_group('text only arguments') @@ -723,7 +721,7 @@ def main(args): parser.add_argument( "-L", "--locale", default=None, - help="locale to be used from month and weekday names" + help="locale to use for month and weekday names" ) parser.add_argument( "-e", "--encoding", @@ -747,7 +745,7 @@ def main(args): help="month number (1-12, text only)" ) - options = parser.parse_args(args[1:]) + options = parser.parse_args(args) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") @@ -756,6 +754,9 @@ def main(args): locale = options.locale, options.encoding if options.type == "html": + if options.month: + parser.error("incorrect number of arguments") + sys.exit(1) if options.locale: cal = LocaleHTMLCalendar(locale=locale) else: @@ -767,11 +768,8 @@ def main(args): write = sys.stdout.buffer.write if options.year is None: write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif options.month is None: - write(cal.formatyearpage(options.year, **optdict)) else: - parser.error("incorrect number of arguments") - sys.exit(1) + write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: cal = LocaleTextCalendar(locale=locale) @@ -795,4 +793,4 @@ def main(args): if __name__ == "__main__": - main(sys.argv) + main() diff --git a/Lib/cgi.py b/Lib/cgi.py deleted file mode 100755 index 8787567be7c081..00000000000000 --- a/Lib/cgi.py +++ /dev/null @@ -1,1012 +0,0 @@ -#! /usr/local/bin/python - -# NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is -# intentionally NOT "/usr/bin/env python". On many systems -# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI -# scripts, and /usr/local/bin is the default directory where Python is -# installed, so /usr/bin/env would be unable to find python. Granted, -# binary installations by Linux vendors often install Python in -# /usr/bin. So let those vendors patch cgi.py to match their choice -# of installation. - -"""Support module for CGI (Common Gateway Interface) scripts. - -This module defines a number of utilities for use by CGI scripts -written in Python. - -The global variable maxlen can be set to an integer indicating the maximum size -of a POST request. POST requests larger than this size will result in a -ValueError being raised during parsing. The default value of this variable is 0, -meaning the request size is unlimited. -""" - -# History -# ------- -# -# Michael McLay started this module. Steve Majewski changed the -# interface to SvFormContentDict and FormContentDict. The multipart -# parsing was inspired by code submitted by Andreas Paepcke. Guido van -# Rossum rewrote, reformatted and documented the module and is currently -# responsible for its maintenance. -# - -__version__ = "2.6" - - -# Imports -# ======= - -from io import StringIO, BytesIO, TextIOWrapper -from collections.abc import Mapping -import sys -import os -import urllib.parse -from email.parser import FeedParser -from email.message import Message -import html -import locale -import tempfile -import warnings - -__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart", - "parse_header", "test", "print_exception", "print_environ", - "print_form", "print_directory", "print_arguments", - "print_environ_usage"] - - -warnings._deprecated(__name__, remove=(3,13)) - -# Logging support -# =============== - -logfile = "" # Filename to log to, if not empty -logfp = None # File object to log to, if not None - -def initlog(*allargs): - """Write a log message, if there is a log file. - - Even though this function is called initlog(), you should always - use log(); log is a variable that is set either to initlog - (initially), to dolog (once the log file has been opened), or to - nolog (when logging is disabled). - - The first argument is a format string; the remaining arguments (if - any) are arguments to the % operator, so e.g. - log("%s: %s", "a", "b") - will write "a: b" to the log file, followed by a newline. - - If the global logfp is not None, it should be a file object to - which log data is written. - - If the global logfp is None, the global logfile may be a string - giving a filename to open, in append mode. This file should be - world writable!!! If the file can't be opened, logging is - silently disabled (since there is no safe place where we could - send an error message). - - """ - global log, logfile, logfp - warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead", - DeprecationWarning, stacklevel=2) - if logfile and not logfp: - try: - logfp = open(logfile, "a", encoding="locale") - except OSError: - pass - if not logfp: - log = nolog - else: - log = dolog - log(*allargs) - -def dolog(fmt, *args): - """Write a log message to the log file. See initlog() for docs.""" - logfp.write(fmt%args + "\n") - -def nolog(*allargs): - """Dummy function, assigned to log when logging is disabled.""" - pass - -def closelog(): - """Close the log file.""" - global log, logfile, logfp - logfile = '' - if logfp: - logfp.close() - logfp = None - log = initlog - -log = initlog # The current logging function - - -# Parsing functions -# ================= - -# Maximum input we will accept when REQUEST_METHOD is POST -# 0 ==> unlimited input -maxlen = 0 - -def parse(fp=None, environ=os.environ, keep_blank_values=0, - strict_parsing=0, separator='&'): - """Parse a query in the environment or from a file (default stdin) - - Arguments, all optional: - - fp : file pointer; default: sys.stdin.buffer - - environ : environment dictionary; default: os.environ - - keep_blank_values: flag indicating whether blank values in - percent-encoded forms should be treated as blank strings. - A true value indicates that blanks should be retained as - blank strings. The default false value indicates that - blank values are to be ignored and treated as if they were - not included. - - strict_parsing: flag indicating what to do with parsing errors. - If false (the default), errors are silently ignored. - If true, errors raise a ValueError exception. - - separator: str. The symbol to use for separating the query arguments. - Defaults to &. - """ - if fp is None: - fp = sys.stdin - - # field keys and values (except for files) are returned as strings - # an encoding is required to decode the bytes read from self.fp - if hasattr(fp,'encoding'): - encoding = fp.encoding - else: - encoding = 'latin-1' - - # fp.read() must return bytes - if isinstance(fp, TextIOWrapper): - fp = fp.buffer - - if not 'REQUEST_METHOD' in environ: - environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone - if environ['REQUEST_METHOD'] == 'POST': - ctype, pdict = parse_header(environ['CONTENT_TYPE']) - if ctype == 'multipart/form-data': - return parse_multipart(fp, pdict, separator=separator) - elif ctype == 'application/x-www-form-urlencoded': - clength = int(environ['CONTENT_LENGTH']) - if maxlen and clength > maxlen: - raise ValueError('Maximum content length exceeded') - qs = fp.read(clength).decode(encoding) - else: - qs = '' # Unknown content-type - if 'QUERY_STRING' in environ: - if qs: qs = qs + '&' - qs = qs + environ['QUERY_STRING'] - elif sys.argv[1:]: - if qs: qs = qs + '&' - qs = qs + sys.argv[1] - environ['QUERY_STRING'] = qs # XXX Shouldn't, really - elif 'QUERY_STRING' in environ: - qs = environ['QUERY_STRING'] - else: - if sys.argv[1:]: - qs = sys.argv[1] - else: - qs = "" - environ['QUERY_STRING'] = qs # XXX Shouldn't, really - return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, - encoding=encoding, separator=separator) - - -def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): - """Parse multipart input. - - Arguments: - fp : input file - pdict: dictionary containing other parameters of content-type header - encoding, errors: request encoding and error handler, passed to - FieldStorage - - Returns a dictionary just like parse_qs(): keys are the field names, each - value is a list of values for that field. For non-file fields, the value - is a list of strings. - """ - # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always - # represented as 7bit US-ASCII. - boundary = pdict['boundary'].decode('ascii') - ctype = "multipart/form-data; boundary={}".format(boundary) - headers = Message() - headers.set_type(ctype) - try: - headers['Content-Length'] = pdict['CONTENT-LENGTH'] - except KeyError: - pass - fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, - environ={'REQUEST_METHOD': 'POST'}, separator=separator) - return {k: fs.getlist(k) for k in fs} - -def _parseparam(s): - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) - if end < 0: - end = len(s) - f = s[:end] - yield f.strip() - s = s[end:] - -def parse_header(line): - """Parse a Content-type like header. - - Return the main content-type and a dictionary of options. - - """ - parts = _parseparam(';' + line) - key = parts.__next__() - pdict = {} - for p in parts: - i = p.find('=') - if i >= 0: - name = p[:i].strip().lower() - value = p[i+1:].strip() - if len(value) >= 2 and value[0] == value[-1] == '"': - value = value[1:-1] - value = value.replace('\\\\', '\\').replace('\\"', '"') - pdict[name] = value - return key, pdict - - -# Classes for field storage -# ========================= - -class MiniFieldStorage: - - """Like FieldStorage, for use when no file uploads are possible.""" - - # Dummy attributes - filename = None - list = None - type = None - file = None - type_options = {} - disposition = None - disposition_options = {} - headers = {} - - def __init__(self, name, value): - """Constructor from field name and value.""" - self.name = name - self.value = value - # self.file = StringIO(value) - - def __repr__(self): - """Return printable representation.""" - return "MiniFieldStorage(%r, %r)" % (self.name, self.value) - - -class FieldStorage: - - """Store a sequence of fields, reading multipart/form-data. - - This class provides naming, typing, files stored on disk, and - more. At the top level, it is accessible like a dictionary, whose - keys are the field names. (Note: None can occur as a field name.) - The items are either a Python list (if there's multiple values) or - another FieldStorage or MiniFieldStorage object. If it's a single - object, it has the following attributes: - - name: the field name, if specified; otherwise None - - filename: the filename, if specified; otherwise None; this is the - client side filename, *not* the file name on which it is - stored (that's a temporary file you don't deal with) - - value: the value as a *string*; for file uploads, this - transparently reads the file every time you request the value - and returns *bytes* - - file: the file(-like) object from which you can read the data *as - bytes* ; None if the data is stored a simple string - - type: the content-type, or None if not specified - - type_options: dictionary of options specified on the content-type - line - - disposition: content-disposition, or None if not specified - - disposition_options: dictionary of corresponding options - - headers: a dictionary(-like) object (sometimes email.message.Message or a - subclass thereof) containing *all* headers - - The class is subclassable, mostly for the purpose of overriding - the make_file() method, which is called internally to come up with - a file open for reading and writing. This makes it possible to - override the default choice of storing all files in a temporary - directory and unlinking them as soon as they have been opened. - - """ - def __init__(self, fp=None, headers=None, outerboundary=b'', - environ=os.environ, keep_blank_values=0, strict_parsing=0, - limit=None, encoding='utf-8', errors='replace', - max_num_fields=None, separator='&'): - """Constructor. Read multipart/* until last part. - - Arguments, all optional: - - fp : file pointer; default: sys.stdin.buffer - (not used when the request method is GET) - Can be : - 1. a TextIOWrapper object - 2. an object whose read() and readline() methods return bytes - - headers : header dictionary-like object; default: - taken from environ as per CGI spec - - outerboundary : terminating multipart boundary - (for internal use only) - - environ : environment dictionary; default: os.environ - - keep_blank_values: flag indicating whether blank values in - percent-encoded forms should be treated as blank strings. - A true value indicates that blanks should be retained as - blank strings. The default false value indicates that - blank values are to be ignored and treated as if they were - not included. - - strict_parsing: flag indicating what to do with parsing errors. - If false (the default), errors are silently ignored. - If true, errors raise a ValueError exception. - - limit : used internally to read parts of multipart/form-data forms, - to exit from the reading loop when reached. It is the difference - between the form content-length and the number of bytes already - read - - encoding, errors : the encoding and error handler used to decode the - binary stream to strings. Must be the same as the charset defined - for the page sending the form (content-type : meta http-equiv or - header) - - max_num_fields: int. If set, then __init__ throws a ValueError - if there are more than n fields read by parse_qsl(). - - """ - method = 'GET' - self.keep_blank_values = keep_blank_values - self.strict_parsing = strict_parsing - self.max_num_fields = max_num_fields - self.separator = separator - if 'REQUEST_METHOD' in environ: - method = environ['REQUEST_METHOD'].upper() - self.qs_on_post = None - if method == 'GET' or method == 'HEAD': - if 'QUERY_STRING' in environ: - qs = environ['QUERY_STRING'] - elif sys.argv[1:]: - qs = sys.argv[1] - else: - qs = "" - qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape') - fp = BytesIO(qs) - if headers is None: - headers = {'content-type': - "application/x-www-form-urlencoded"} - if headers is None: - headers = {} - if method == 'POST': - # Set default content-type for POST to what's traditional - headers['content-type'] = "application/x-www-form-urlencoded" - if 'CONTENT_TYPE' in environ: - headers['content-type'] = environ['CONTENT_TYPE'] - if 'QUERY_STRING' in environ: - self.qs_on_post = environ['QUERY_STRING'] - if 'CONTENT_LENGTH' in environ: - headers['content-length'] = environ['CONTENT_LENGTH'] - else: - if not (isinstance(headers, (Mapping, Message))): - raise TypeError("headers must be mapping or an instance of " - "email.message.Message") - self.headers = headers - if fp is None: - self.fp = sys.stdin.buffer - # self.fp.read() must return bytes - elif isinstance(fp, TextIOWrapper): - self.fp = fp.buffer - else: - if not (hasattr(fp, 'read') and hasattr(fp, 'readline')): - raise TypeError("fp must be file pointer") - self.fp = fp - - self.encoding = encoding - self.errors = errors - - if not isinstance(outerboundary, bytes): - raise TypeError('outerboundary must be bytes, not %s' - % type(outerboundary).__name__) - self.outerboundary = outerboundary - - self.bytes_read = 0 - self.limit = limit - - # Process content-disposition header - cdisp, pdict = "", {} - if 'content-disposition' in self.headers: - cdisp, pdict = parse_header(self.headers['content-disposition']) - self.disposition = cdisp - self.disposition_options = pdict - self.name = None - if 'name' in pdict: - self.name = pdict['name'] - self.filename = None - if 'filename' in pdict: - self.filename = pdict['filename'] - self._binary_file = self.filename is not None - - # Process content-type header - # - # Honor any existing content-type header. But if there is no - # content-type header, use some sensible defaults. Assume - # outerboundary is "" at the outer level, but something non-false - # inside a multi-part. The default for an inner part is text/plain, - # but for an outer part it should be urlencoded. This should catch - # bogus clients which erroneously forget to include a content-type - # header. - # - # See below for what we do if there does exist a content-type header, - # but it happens to be something we don't understand. - if 'content-type' in self.headers: - ctype, pdict = parse_header(self.headers['content-type']) - elif self.outerboundary or method != 'POST': - ctype, pdict = "text/plain", {} - else: - ctype, pdict = 'application/x-www-form-urlencoded', {} - self.type = ctype - self.type_options = pdict - if 'boundary' in pdict: - self.innerboundary = pdict['boundary'].encode(self.encoding, - self.errors) - else: - self.innerboundary = b"" - - clen = -1 - if 'content-length' in self.headers: - try: - clen = int(self.headers['content-length']) - except ValueError: - pass - if maxlen and clen > maxlen: - raise ValueError('Maximum content length exceeded') - self.length = clen - if self.limit is None and clen >= 0: - self.limit = clen - - self.list = self.file = None - self.done = 0 - if ctype == 'application/x-www-form-urlencoded': - self.read_urlencoded() - elif ctype[:10] == 'multipart/': - self.read_multi(environ, keep_blank_values, strict_parsing) - else: - self.read_single() - - def __del__(self): - try: - self.file.close() - except AttributeError: - pass - - def __enter__(self): - return self - - def __exit__(self, *args): - self.file.close() - - def __repr__(self): - """Return a printable representation.""" - return "FieldStorage(%r, %r, %r)" % ( - self.name, self.filename, self.value) - - def __iter__(self): - return iter(self.keys()) - - def __getattr__(self, name): - if name != 'value': - raise AttributeError(name) - if self.file: - self.file.seek(0) - value = self.file.read() - self.file.seek(0) - elif self.list is not None: - value = self.list - else: - value = None - return value - - def __getitem__(self, key): - """Dictionary style indexing.""" - if self.list is None: - raise TypeError("not indexable") - found = [] - for item in self.list: - if item.name == key: found.append(item) - if not found: - raise KeyError(key) - if len(found) == 1: - return found[0] - else: - return found - - def getvalue(self, key, default=None): - """Dictionary style get() method, including 'value' lookup.""" - if key in self: - value = self[key] - if isinstance(value, list): - return [x.value for x in value] - else: - return value.value - else: - return default - - def getfirst(self, key, default=None): - """ Return the first value received.""" - if key in self: - value = self[key] - if isinstance(value, list): - return value[0].value - else: - return value.value - else: - return default - - def getlist(self, key): - """ Return list of received values.""" - if key in self: - value = self[key] - if isinstance(value, list): - return [x.value for x in value] - else: - return [value.value] - else: - return [] - - def keys(self): - """Dictionary style keys() method.""" - if self.list is None: - raise TypeError("not indexable") - return list(set(item.name for item in self.list)) - - def __contains__(self, key): - """Dictionary style __contains__ method.""" - if self.list is None: - raise TypeError("not indexable") - return any(item.name == key for item in self.list) - - def __len__(self): - """Dictionary style len(x) support.""" - return len(self.keys()) - - def __bool__(self): - if self.list is None: - raise TypeError("Cannot be converted to bool.") - return bool(self.list) - - def read_urlencoded(self): - """Internal: read data in query string format.""" - qs = self.fp.read(self.length) - if not isinstance(qs, bytes): - raise ValueError("%s should return bytes, got %s" \ - % (self.fp, type(qs).__name__)) - qs = qs.decode(self.encoding, self.errors) - if self.qs_on_post: - qs += '&' + self.qs_on_post - query = urllib.parse.parse_qsl( - qs, self.keep_blank_values, self.strict_parsing, - encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields, separator=self.separator) - self.list = [MiniFieldStorage(key, value) for key, value in query] - self.skip_lines() - - FieldStorageClass = None - - def read_multi(self, environ, keep_blank_values, strict_parsing): - """Internal: read a part that is itself multipart.""" - ib = self.innerboundary - if not valid_boundary(ib): - raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) - self.list = [] - if self.qs_on_post: - query = urllib.parse.parse_qsl( - self.qs_on_post, self.keep_blank_values, self.strict_parsing, - encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields, separator=self.separator) - self.list.extend(MiniFieldStorage(key, value) for key, value in query) - - klass = self.FieldStorageClass or self.__class__ - first_line = self.fp.readline() # bytes - if not isinstance(first_line, bytes): - raise ValueError("%s should return bytes, got %s" \ - % (self.fp, type(first_line).__name__)) - self.bytes_read += len(first_line) - - # Ensure that we consume the file until we've hit our inner boundary - while (first_line.strip() != (b"--" + self.innerboundary) and - first_line): - first_line = self.fp.readline() - self.bytes_read += len(first_line) - - # Propagate max_num_fields into the sub class appropriately - max_num_fields = self.max_num_fields - if max_num_fields is not None: - max_num_fields -= len(self.list) - - while True: - parser = FeedParser() - hdr_text = b"" - while True: - data = self.fp.readline() - hdr_text += data - if not data.strip(): - break - if not hdr_text: - break - # parser takes strings, not bytes - self.bytes_read += len(hdr_text) - parser.feed(hdr_text.decode(self.encoding, self.errors)) - headers = parser.close() - - # Some clients add Content-Length for part headers, ignore them - if 'content-length' in headers: - del headers['content-length'] - - limit = None if self.limit is None \ - else self.limit - self.bytes_read - part = klass(self.fp, headers, ib, environ, keep_blank_values, - strict_parsing, limit, - self.encoding, self.errors, max_num_fields, self.separator) - - if max_num_fields is not None: - max_num_fields -= 1 - if part.list: - max_num_fields -= len(part.list) - if max_num_fields < 0: - raise ValueError('Max number of fields exceeded') - - self.bytes_read += part.bytes_read - self.list.append(part) - if part.done or self.bytes_read >= self.length > 0: - break - self.skip_lines() - - def read_single(self): - """Internal: read an atomic part.""" - if self.length >= 0: - self.read_binary() - self.skip_lines() - else: - self.read_lines() - self.file.seek(0) - - bufsize = 8*1024 # I/O buffering size for copy to file - - def read_binary(self): - """Internal: read binary data.""" - self.file = self.make_file() - todo = self.length - if todo >= 0: - while todo > 0: - data = self.fp.read(min(todo, self.bufsize)) # bytes - if not isinstance(data, bytes): - raise ValueError("%s should return bytes, got %s" - % (self.fp, type(data).__name__)) - self.bytes_read += len(data) - if not data: - self.done = -1 - break - self.file.write(data) - todo = todo - len(data) - - def read_lines(self): - """Internal: read lines until EOF or outerboundary.""" - if self._binary_file: - self.file = self.__file = BytesIO() # store data as bytes for files - else: - self.file = self.__file = StringIO() # as strings for other fields - if self.outerboundary: - self.read_lines_to_outerboundary() - else: - self.read_lines_to_eof() - - def __write(self, line): - """line is always bytes, not string""" - if self.__file is not None: - if self.__file.tell() + len(line) > 1000: - self.file = self.make_file() - data = self.__file.getvalue() - self.file.write(data) - self.__file = None - if self._binary_file: - # keep bytes - self.file.write(line) - else: - # decode to string - self.file.write(line.decode(self.encoding, self.errors)) - - def read_lines_to_eof(self): - """Internal: read lines until EOF.""" - while 1: - line = self.fp.readline(1<<16) # bytes - self.bytes_read += len(line) - if not line: - self.done = -1 - break - self.__write(line) - - def read_lines_to_outerboundary(self): - """Internal: read lines until outerboundary. - Data is read as bytes: boundaries and line ends must be converted - to bytes for comparisons. - """ - next_boundary = b"--" + self.outerboundary - last_boundary = next_boundary + b"--" - delim = b"" - last_line_lfend = True - _read = 0 - while 1: - - if self.limit is not None and 0 <= self.limit <= _read: - break - line = self.fp.readline(1<<16) # bytes - self.bytes_read += len(line) - _read += len(line) - if not line: - self.done = -1 - break - if delim == b"\r": - line = delim + line - delim = b"" - if line.startswith(b"--") and last_line_lfend: - strippedline = line.rstrip() - if strippedline == next_boundary: - break - if strippedline == last_boundary: - self.done = 1 - break - odelim = delim - if line.endswith(b"\r\n"): - delim = b"\r\n" - line = line[:-2] - last_line_lfend = True - elif line.endswith(b"\n"): - delim = b"\n" - line = line[:-1] - last_line_lfend = True - elif line.endswith(b"\r"): - # We may interrupt \r\n sequences if they span the 2**16 - # byte boundary - delim = b"\r" - line = line[:-1] - last_line_lfend = False - else: - delim = b"" - last_line_lfend = False - self.__write(odelim + line) - - def skip_lines(self): - """Internal: skip lines until outer boundary if defined.""" - if not self.outerboundary or self.done: - return - next_boundary = b"--" + self.outerboundary - last_boundary = next_boundary + b"--" - last_line_lfend = True - while True: - line = self.fp.readline(1<<16) - self.bytes_read += len(line) - if not line: - self.done = -1 - break - if line.endswith(b"--") and last_line_lfend: - strippedline = line.strip() - if strippedline == next_boundary: - break - if strippedline == last_boundary: - self.done = 1 - break - last_line_lfend = line.endswith(b'\n') - - def make_file(self): - """Overridable: return a readable & writable file. - - The file will be used as follows: - - data is written to it - - seek(0) - - data is read from it - - The file is opened in binary mode for files, in text mode - for other fields - - This version opens a temporary file for reading and writing, - and immediately deletes (unlinks) it. The trick (on Unix!) is - that the file can still be used, but it can't be opened by - another process, and it will automatically be deleted when it - is closed or when the current process terminates. - - If you want a more permanent file, you derive a class which - overrides this method. If you want a visible temporary file - that is nevertheless automatically deleted when the script - terminates, try defining a __del__ method in a derived class - which unlinks the temporary files you have created. - - """ - if self._binary_file: - return tempfile.TemporaryFile("wb+") - else: - return tempfile.TemporaryFile("w+", - encoding=self.encoding, newline = '\n') - - -# Test/debug code -# =============== - -def test(environ=os.environ): - """Robust test CGI script, usable as main program. - - Write minimal HTTP headers and dump all information provided to - the script in HTML form. - - """ - print("Content-type: text/html") - print() - sys.stderr = sys.stdout - try: - form = FieldStorage() # Replace with other classes to test those - print_directory() - print_arguments() - print_form(form) - print_environ(environ) - print_environ_usage() - def f(): - exec("testing print_exception() -- italics?") - def g(f=f): - f() - print("

What follows is a test, not an actual exception:

") - g() - except: - print_exception() - - print("

Second try with a small maxlen...

") - - global maxlen - maxlen = 50 - try: - form = FieldStorage() # Replace with other classes to test those - print_directory() - print_arguments() - print_form(form) - print_environ(environ) - except: - print_exception() - -def print_exception(type=None, value=None, tb=None, limit=None): - if type is None: - type, value, tb = sys.exc_info() - import traceback - print() - print("

Traceback (most recent call last):

") - list = traceback.format_tb(tb, limit) + \ - traceback.format_exception_only(type, value) - print("
%s%s
" % ( - html.escape("".join(list[:-1])), - html.escape(list[-1]), - )) - del tb - -def print_environ(environ=os.environ): - """Dump the shell environment as HTML.""" - keys = sorted(environ.keys()) - print() - print("

Shell Environment:

") - print("
") - for key in keys: - print("
", html.escape(key), "
", html.escape(environ[key])) - print("
") - print() - -def print_form(form): - """Dump the contents of a form as HTML.""" - keys = sorted(form.keys()) - print() - print("

Form Contents:

") - if not keys: - print("

No form fields.") - print("

") - for key in keys: - print("
" + html.escape(key) + ":", end=' ') - value = form[key] - print("" + html.escape(repr(type(value))) + "") - print("
" + html.escape(repr(value))) - print("
") - print() - -def print_directory(): - """Dump the current directory as HTML.""" - print() - print("

Current Working Directory:

") - try: - pwd = os.getcwd() - except OSError as msg: - print("OSError:", html.escape(str(msg))) - else: - print(html.escape(pwd)) - print() - -def print_arguments(): - print() - print("

Command Line Arguments:

") - print() - print(sys.argv) - print() - -def print_environ_usage(): - """Dump a list of environment variables used by CGI as HTML.""" - print(""" -

These environment variables could have been set:

-
    -
  • AUTH_TYPE -
  • CONTENT_LENGTH -
  • CONTENT_TYPE -
  • DATE_GMT -
  • DATE_LOCAL -
  • DOCUMENT_NAME -
  • DOCUMENT_ROOT -
  • DOCUMENT_URI -
  • GATEWAY_INTERFACE -
  • LAST_MODIFIED -
  • PATH -
  • PATH_INFO -
  • PATH_TRANSLATED -
  • QUERY_STRING -
  • REMOTE_ADDR -
  • REMOTE_HOST -
  • REMOTE_IDENT -
  • REMOTE_USER -
  • REQUEST_METHOD -
  • SCRIPT_NAME -
  • SERVER_NAME -
  • SERVER_PORT -
  • SERVER_PROTOCOL -
  • SERVER_ROOT -
  • SERVER_SOFTWARE -
-In addition, HTTP headers sent by the server may be passed in the -environment as well. Here are some common variable names: -
    -
  • HTTP_ACCEPT -
  • HTTP_CONNECTION -
  • HTTP_HOST -
  • HTTP_PRAGMA -
  • HTTP_REFERER -
  • HTTP_USER_AGENT -
-""") - - -# Utilities -# ========= - -def valid_boundary(s): - import re - if isinstance(s, bytes): - _vb_pattern = b"^[ -~]{0,200}[!-~]$" - else: - _vb_pattern = "^[ -~]{0,200}[!-~]$" - return re.match(_vb_pattern, s) - -# Invoke mainline -# =============== - -# Call test() when this file is run as a script (not imported as a module) -if __name__ == '__main__': - test() diff --git a/Lib/cgitb.py b/Lib/cgitb.py deleted file mode 100644 index f6b97f25c59de9..00000000000000 --- a/Lib/cgitb.py +++ /dev/null @@ -1,332 +0,0 @@ -"""More comprehensive traceback formatting for Python scripts. - -To enable this module, do: - - import cgitb; cgitb.enable() - -at the top of your script. The optional arguments to enable() are: - - display - if true, tracebacks are displayed in the web browser - logdir - if set, tracebacks are written to files in this directory - context - number of lines of source code to show for each stack frame - format - 'text' or 'html' controls the output format - -By default, tracebacks are displayed but not saved, the context is 5 lines -and the output format is 'html' (for backwards compatibility with the -original use of this module) - -Alternatively, if you have caught an exception and want cgitb to display it -for you, call cgitb.handler(). The optional argument to handler() is a -3-item tuple (etype, evalue, etb) just like the value of sys.exc_info(). -The default handler displays output as HTML. - -""" -import inspect -import keyword -import linecache -import os -import pydoc -import sys -import tempfile -import time -import tokenize -import traceback -import warnings -from html import escape as html_escape - -warnings._deprecated(__name__, remove=(3, 13)) - - -def reset(): - """Return a string that resets the CGI and browser to a known state.""" - return ''' - --> --> - - ''' - -__UNDEF__ = [] # a special sentinel object -def small(text): - if text: - return '' + text + '' - else: - return '' - -def strong(text): - if text: - return '' + text + '' - else: - return '' - -def grey(text): - if text: - return '' + text + '' - else: - return '' - -def lookup(name, frame, locals): - """Find the value for a given name in the given environment.""" - if name in locals: - return 'local', locals[name] - if name in frame.f_globals: - return 'global', frame.f_globals[name] - if '__builtins__' in frame.f_globals: - builtins = frame.f_globals['__builtins__'] - if isinstance(builtins, dict): - if name in builtins: - return 'builtin', builtins[name] - else: - if hasattr(builtins, name): - return 'builtin', getattr(builtins, name) - return None, __UNDEF__ - -def scanvars(reader, frame, locals): - """Scan one logical line of Python and look up values of variables used.""" - vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__ - for ttype, token, start, end, line in tokenize.generate_tokens(reader): - if ttype == tokenize.NEWLINE: break - if ttype == tokenize.NAME and token not in keyword.kwlist: - if lasttoken == '.': - if parent is not __UNDEF__: - value = getattr(parent, token, __UNDEF__) - vars.append((prefix + token, prefix, value)) - else: - where, value = lookup(token, frame, locals) - vars.append((token, where, value)) - elif token == '.': - prefix += lasttoken + '.' - parent = value - else: - parent, prefix = None, '' - lasttoken = token - return vars - -def html(einfo, context=5): - """Return a nice HTML document describing a given traceback.""" - etype, evalue, etb = einfo - if isinstance(etype, type): - etype = etype.__name__ - pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable - date = time.ctime(time.time()) - head = f''' - - - - - -
 
- 
-{html_escape(str(etype))}
-{pyver}
{date}
-

A problem occurred in a Python script. Here is the sequence of -function calls leading up to the error, in the order they occurred.

''' - - indent = '' + small(' ' * 5) + ' ' - frames = [] - records = inspect.getinnerframes(etb, context) - for frame, file, lnum, func, lines, index in records: - if file: - file = os.path.abspath(file) - link = '%s' % (file, pydoc.html.escape(file)) - else: - file = link = '?' - args, varargs, varkw, locals = inspect.getargvalues(frame) - call = '' - if func != '?': - call = 'in ' + strong(pydoc.html.escape(func)) - if func != "": - call += inspect.formatargvalues(args, varargs, varkw, locals, - formatvalue=lambda value: '=' + pydoc.html.repr(value)) - - highlight = {} - def reader(lnum=[lnum]): - highlight[lnum[0]] = 1 - try: return linecache.getline(file, lnum[0]) - finally: lnum[0] += 1 - vars = scanvars(reader, frame, locals) - - rows = ['%s%s %s' % - (' ', link, call)] - if index is not None: - i = lnum - index - for line in lines: - num = small(' ' * (5-len(str(i))) + str(i)) + ' ' - if i in highlight: - line = '=>%s%s' % (num, pydoc.html.preformat(line)) - rows.append('%s' % line) - else: - line = '  %s%s' % (num, pydoc.html.preformat(line)) - rows.append('%s' % grey(line)) - i += 1 - - done, dump = {}, [] - for name, where, value in vars: - if name in done: continue - done[name] = 1 - if value is not __UNDEF__: - if where in ('global', 'builtin'): - name = ('%s ' % where) + strong(name) - elif where == 'local': - name = strong(name) - else: - name = where + strong(name.split('.')[-1]) - dump.append('%s = %s' % (name, pydoc.html.repr(value))) - else: - dump.append(name + ' undefined') - - rows.append('%s' % small(grey(', '.join(dump)))) - frames.append(''' - -%s
''' % '\n'.join(rows)) - - exception = ['

%s: %s' % (strong(pydoc.html.escape(str(etype))), - pydoc.html.escape(str(evalue)))] - for name in dir(evalue): - if name[:1] == '_': continue - value = pydoc.html.repr(getattr(evalue, name)) - exception.append('\n
%s%s =\n%s' % (indent, name, value)) - - return head + ''.join(frames) + ''.join(exception) + ''' - - - -''' % pydoc.html.escape( - ''.join(traceback.format_exception(etype, evalue, etb))) - -def text(einfo, context=5): - """Return a plain text document describing a given traceback.""" - etype, evalue, etb = einfo - if isinstance(etype, type): - etype = etype.__name__ - pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable - date = time.ctime(time.time()) - head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + ''' -A problem occurred in a Python script. Here is the sequence of -function calls leading up to the error, in the order they occurred. -''' - - frames = [] - records = inspect.getinnerframes(etb, context) - for frame, file, lnum, func, lines, index in records: - file = file and os.path.abspath(file) or '?' - args, varargs, varkw, locals = inspect.getargvalues(frame) - call = '' - if func != '?': - call = 'in ' + func - if func != "": - call += inspect.formatargvalues(args, varargs, varkw, locals, - formatvalue=lambda value: '=' + pydoc.text.repr(value)) - - highlight = {} - def reader(lnum=[lnum]): - highlight[lnum[0]] = 1 - try: return linecache.getline(file, lnum[0]) - finally: lnum[0] += 1 - vars = scanvars(reader, frame, locals) - - rows = [' %s %s' % (file, call)] - if index is not None: - i = lnum - index - for line in lines: - num = '%5d ' % i - rows.append(num+line.rstrip()) - i += 1 - - done, dump = {}, [] - for name, where, value in vars: - if name in done: continue - done[name] = 1 - if value is not __UNDEF__: - if where == 'global': name = 'global ' + name - elif where != 'local': name = where + name.split('.')[-1] - dump.append('%s = %s' % (name, pydoc.text.repr(value))) - else: - dump.append(name + ' undefined') - - rows.append('\n'.join(dump)) - frames.append('\n%s\n' % '\n'.join(rows)) - - exception = ['%s: %s' % (str(etype), str(evalue))] - for name in dir(evalue): - value = pydoc.text.repr(getattr(evalue, name)) - exception.append('\n%s%s = %s' % (" "*4, name, value)) - - return head + ''.join(frames) + ''.join(exception) + ''' - -The above is a description of an error in a Python program. Here is -the original traceback: - -%s -''' % ''.join(traceback.format_exception(etype, evalue, etb)) - -class Hook: - """A hook to replace sys.excepthook that shows tracebacks in HTML.""" - - def __init__(self, display=1, logdir=None, context=5, file=None, - format="html"): - self.display = display # send tracebacks to browser if true - self.logdir = logdir # log tracebacks to files if not None - self.context = context # number of source code lines per frame - self.file = file or sys.stdout # place to send the output - self.format = format - - def __call__(self, etype, evalue, etb): - self.handle((etype, evalue, etb)) - - def handle(self, info=None): - info = info or sys.exc_info() - if self.format == "html": - self.file.write(reset()) - - formatter = (self.format=="html") and html or text - plain = False - try: - doc = formatter(info, self.context) - except: # just in case something goes wrong - doc = ''.join(traceback.format_exception(*info)) - plain = True - - if self.display: - if plain: - doc = pydoc.html.escape(doc) - self.file.write('

' + doc + '
\n') - else: - self.file.write(doc + '\n') - else: - self.file.write('

A problem occurred in a Python script.\n') - - if self.logdir is not None: - suffix = ['.txt', '.html'][self.format=="html"] - (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir) - - try: - with os.fdopen(fd, 'w') as file: - file.write(doc) - msg = '%s contains the description of this error.' % path - except: - msg = 'Tried to save traceback to %s, but failed.' % path - - if self.format == 'html': - self.file.write('

%s

\n' % msg) - else: - self.file.write(msg + '\n') - try: - self.file.flush() - except: pass - -handler = Hook().handle -def enable(display=1, logdir=None, context=5, format="html"): - """Install an exception handler that formats tracebacks as HTML. - - The optional argument 'display' can be set to 0 to suppress sending the - traceback to the browser, and 'logdir' can be set to a directory to cause - tracebacks to be written to files there.""" - sys.excepthook = Hook(display=display, logdir=logdir, - context=context, format=format) diff --git a/Lib/chunk.py b/Lib/chunk.py deleted file mode 100644 index 618781efd11e71..00000000000000 --- a/Lib/chunk.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Simple class to read IFF chunks. - -An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File -Format)) has the following structure: - -+----------------+ -| ID (4 bytes) | -+----------------+ -| size (4 bytes) | -+----------------+ -| data | -| ... | -+----------------+ - -The ID is a 4-byte string which identifies the type of chunk. - -The size field (a 32-bit value, encoded using big-endian byte order) -gives the size of the whole chunk, including the 8-byte header. - -Usually an IFF-type file consists of one or more chunks. The proposed -usage of the Chunk class defined here is to instantiate an instance at -the start of each chunk and read from the instance until it reaches -the end, after which a new instance can be instantiated. At the end -of the file, creating a new instance will fail with an EOFError -exception. - -Usage: -while True: - try: - chunk = Chunk(file) - except EOFError: - break - chunktype = chunk.getname() - while True: - data = chunk.read(nbytes) - if not data: - pass - # do something with data - -The interface is file-like. The implemented methods are: -read, close, seek, tell, isatty. -Extra methods are: skip() (called by close, skips to the end of the chunk), -getname() (returns the name (ID) of the chunk) - -The __init__ method has one required argument, a file-like object -(including a chunk instance), and one optional argument, a flag which -specifies whether or not chunks are aligned on 2-byte boundaries. The -default is 1, i.e. aligned. -""" - -import warnings - -warnings._deprecated(__name__, remove=(3, 13)) - -class Chunk: - def __init__(self, file, align=True, bigendian=True, inclheader=False): - import struct - self.closed = False - self.align = align # whether to align to word (2-byte) boundaries - if bigendian: - strflag = '>' - else: - strflag = '<' - self.file = file - self.chunkname = file.read(4) - if len(self.chunkname) < 4: - raise EOFError - try: - self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0] - except struct.error: - raise EOFError from None - if inclheader: - self.chunksize = self.chunksize - 8 # subtract header - self.size_read = 0 - try: - self.offset = self.file.tell() - except (AttributeError, OSError): - self.seekable = False - else: - self.seekable = True - - def getname(self): - """Return the name (ID) of the current chunk.""" - return self.chunkname - - def getsize(self): - """Return the size of the current chunk.""" - return self.chunksize - - def close(self): - if not self.closed: - try: - self.skip() - finally: - self.closed = True - - def isatty(self): - if self.closed: - raise ValueError("I/O operation on closed file") - return False - - def seek(self, pos, whence=0): - """Seek to specified position into the chunk. - Default position is 0 (start of chunk). - If the file is not seekable, this will result in an error. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if not self.seekable: - raise OSError("cannot seek") - if whence == 1: - pos = pos + self.size_read - elif whence == 2: - pos = pos + self.chunksize - if pos < 0 or pos > self.chunksize: - raise RuntimeError - self.file.seek(self.offset + pos, 0) - self.size_read = pos - - def tell(self): - if self.closed: - raise ValueError("I/O operation on closed file") - return self.size_read - - def read(self, size=-1): - """Read at most size bytes from the chunk. - If size is omitted or negative, read until the end - of the chunk. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if self.size_read >= self.chunksize: - return b'' - if size < 0: - size = self.chunksize - self.size_read - if size > self.chunksize - self.size_read: - size = self.chunksize - self.size_read - data = self.file.read(size) - self.size_read = self.size_read + len(data) - if self.size_read == self.chunksize and \ - self.align and \ - (self.chunksize & 1): - dummy = self.file.read(1) - self.size_read = self.size_read + len(dummy) - return data - - def skip(self): - """Skip the rest of the chunk. - If you are not interested in the contents of the chunk, - this method should be called so that the file points to - the start of the next chunk. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if self.seekable: - try: - n = self.chunksize - self.size_read - # maybe fix alignment - if self.align and (self.chunksize & 1): - n = n + 1 - self.file.seek(n, 1) - self.size_read = self.size_read + n - return - except OSError: - pass - while self.size_read < self.chunksize: - n = min(8192, self.chunksize - self.size_read) - dummy = self.read(n) - if not dummy: - raise EOFError diff --git a/Lib/cmd.py b/Lib/cmd.py index 88ee7d3ddc4694..2e358d6cd5a02d 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -108,7 +108,15 @@ def cmdloop(self, intro=None): import readline self.old_completer = readline.get_completer() readline.set_completer(self.complete) - readline.parse_and_bind(self.completekey+": complete") + if readline.backend == "editline": + if self.completekey == 'tab': + # libedit uses "^I" instead of "tab" + command_string = "bind ^I rl_complete" + else: + command_string = f"bind {self.completekey} rl_complete" + else: + command_string = f"{self.completekey}: complete" + readline.parse_and_bind(command_string) except ImportError: pass try: @@ -210,9 +218,8 @@ def onecmd(self, line): if cmd == '': return self.default(line) else: - try: - func = getattr(self, 'do_' + cmd) - except AttributeError: + func = getattr(self, 'do_' + cmd, None) + if func is None: return self.default(line) return func(arg) diff --git a/Lib/code.py b/Lib/code.py index 2bd5fa3e795a61..f4aecddeca7813 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -5,6 +5,7 @@ # Inspired by similar code by Jeff Epler and Fredrik Lundh. +import builtins import sys import traceback from codeop import CommandCompiler, compile_command @@ -169,7 +170,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename=""): + def __init__(self, locals=None, filename="", local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -181,6 +182,7 @@ def __init__(self, locals=None, filename=""): """ InteractiveInterpreter.__init__(self, locals) self.filename = filename + self.local_exit = local_exit self.resetbuffer() def resetbuffer(self): @@ -219,27 +221,64 @@ def interact(self, banner=None, exitmsg=None): elif banner: self.write("%s\n" % str(banner)) more = 0 - while 1: - try: - if more: - prompt = sys.ps2 - else: - prompt = sys.ps1 + + # When the user uses exit() or quit() in their interactive shell + # they probably just want to exit the created shell, not the whole + # process. exit and quit in builtins closes sys.stdin which makes + # it super difficult to restore + # + # When self.local_exit is True, we overwrite the builtins so + # exit() and quit() only raises SystemExit and we can catch that + # to only exit the interactive shell + + _exit = None + _quit = None + + if self.local_exit: + if hasattr(builtins, "exit"): + _exit = builtins.exit + builtins.exit = Quitter("exit") + + if hasattr(builtins, "quit"): + _quit = builtins.quit + builtins.quit = Quitter("quit") + + try: + while True: try: - line = self.raw_input(prompt) - except EOFError: - self.write("\n") - break - else: - more = self.push(line) - except KeyboardInterrupt: - self.write("\nKeyboardInterrupt\n") - self.resetbuffer() - more = 0 - if exitmsg is None: - self.write('now exiting %s...\n' % self.__class__.__name__) - elif exitmsg != '': - self.write('%s\n' % exitmsg) + if more: + prompt = sys.ps2 + else: + prompt = sys.ps1 + try: + line = self.raw_input(prompt) + except EOFError: + self.write("\n") + break + else: + more = self.push(line) + except KeyboardInterrupt: + self.write("\nKeyboardInterrupt\n") + self.resetbuffer() + more = 0 + except SystemExit as e: + if self.local_exit: + self.write("\n") + break + else: + raise e + finally: + # restore exit and quit in builtins if they were modified + if _exit is not None: + builtins.exit = _exit + + if _quit is not None: + builtins.quit = _quit + + if exitmsg is None: + self.write('now exiting %s...\n' % self.__class__.__name__) + elif exitmsg != '': + self.write('%s\n' % exitmsg) def push(self, line): """Push a line to the interpreter. @@ -276,8 +315,22 @@ def raw_input(self, prompt=""): return input(prompt) +class Quitter: + def __init__(self, name): + self.name = name + if sys.platform == "win32": + self.eof = 'Ctrl-Z plus Return' + else: + self.eof = 'Ctrl-D (i.e. EOF)' + + def __repr__(self): + return f'Use {self.name} or {self.eof} to exit' + + def __call__(self, code=None): + raise SystemExit(code) + -def interact(banner=None, readfunc=None, local=None, exitmsg=None): +def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False): """Closely emulate the interactive Python interpreter. This is a backwards compatible interface to the InteractiveConsole @@ -290,9 +343,10 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): readfunc -- if not None, replaces InteractiveConsole.raw_input() local -- passed to InteractiveInterpreter.__init__() exitmsg -- passed to InteractiveConsole.interact() + local_exit -- passed to InteractiveConsole.__init__() """ - console = InteractiveConsole(local) + console = InteractiveConsole(local, local_exit=local_exit) if readfunc is not None: console.raw_input = readfunc else: diff --git a/Lib/codecs.py b/Lib/codecs.py index c1c55d8afef389..9b35b6127dd01c 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -111,6 +111,9 @@ def __repr__(self): (self.__class__.__module__, self.__class__.__qualname__, self.name, id(self)) + def __getnewargs__(self): + return tuple(self) + class Codec: """ Defines the interface for stateless encoders/decoders. @@ -414,6 +417,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamReader(Codec): @@ -663,6 +669,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamReaderWriter: @@ -750,6 +759,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamRecoder: @@ -866,6 +878,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### Shortcuts def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): diff --git a/Lib/codeop.py b/Lib/codeop.py index 2213b69f231f92..91146be2c438e2 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -70,20 +70,14 @@ def _maybe_compile(compiler, source, filename, symbol): return None # fallthrough - return compiler(source, filename, symbol) + return compiler(source, filename, symbol, incomplete_input=False) - -def _is_syntax_error(err1, err2): - rep1 = repr(err1) - rep2 = repr(err2) - if "was never closed" in rep1 and "was never closed" in rep2: - return False - if rep1 == rep2: - return True - return False - -def _compile(source, filename, symbol): - return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT) +def _compile(source, filename, symbol, incomplete_input=True): + flags = 0 + if incomplete_input: + flags |= PyCF_ALLOW_INCOMPLETE_INPUT + flags |= PyCF_DONT_IMPLY_DEDENT + return compile(source, filename, symbol, flags) def compile_command(source, filename="", symbol="single"): r"""Compile a command and determine whether it is incomplete. @@ -114,8 +108,12 @@ class Compile: def __init__(self): self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT - def __call__(self, source, filename, symbol): - codeob = compile(source, filename, symbol, self.flags, True) + def __call__(self, source, filename, symbol, **kwargs): + flags = self.flags + if kwargs.get('incomplete_input', True) is False: + flags &= ~PyCF_DONT_IMPLY_DEDENT + flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT + codeob = compile(source, filename, symbol, flags, True) for feature in _features: if codeob.co_flags & feature.compiler_flag: self.flags |= feature.compiler_flag diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 03ca2d7e18f6f0..2e527dfd810c43 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -95,17 +95,19 @@ class OrderedDict(dict): # Individual links are kept alive by the hard reference in self.__map. # Those hard references disappear when a key is deleted from an OrderedDict. + def __new__(cls, /, *args, **kwds): + "Create the ordered dict object and set up the underlying structures." + self = dict.__new__(cls) + self.__hardroot = _Link() + self.__root = root = _proxy(self.__hardroot) + root.prev = root.next = root + self.__map = {} + return self + def __init__(self, other=(), /, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries. Keyword argument order is preserved. ''' - try: - self.__root - except AttributeError: - self.__hardroot = _Link() - self.__root = root = _proxy(self.__hardroot) - root.prev = root.next = root - self.__map = {} self.__update(other, **kwds) def __setitem__(self, key, value, @@ -455,7 +457,7 @@ def _make(cls, iterable): def _replace(self, /, **kwds): result = self._make(_map(kwds.pop, field_names, self)) if kwds: - raise ValueError(f'Got unexpected field names: {list(kwds)!r}') + raise TypeError(f'Got unexpected field names: {list(kwds)!r}') return result _replace.__doc__ = (f'Return a new {typename} object replacing specified ' @@ -493,6 +495,7 @@ def __getnewargs__(self): '_field_defaults': field_defaults, '__new__': __new__, '_make': _make, + '__replace__': _replace, '_replace': _replace, '__repr__': __repr__, '_asdict': _asdict, diff --git a/Lib/colorsys.py b/Lib/colorsys.py index 9bdc83e3772603..bc897bd0f99298 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -83,7 +83,7 @@ def rgb_to_hls(r, g, b): if l <= 0.5: s = rangec / sumc else: - s = rangec / (2.0-sumc) + s = rangec / (2.0-maxc-minc) # Not always 2.0-sumc: gh-106498. rc = (maxc-r) / rangec gc = (maxc-g) / rangec bc = (maxc-b) / rangec diff --git a/Lib/compileall.py b/Lib/compileall.py index d394156cedc4e7..9b53086bf41380 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -172,13 +172,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) - ddir_parts = list(fullname_parts) - for spart, opart in zip(stripdir_parts, fullname_parts): - if spart == opart: - ddir_parts.remove(spart) - - dfile = os.path.join(*ddir_parts) + if stripdir_parts != fullname_parts[:len(stripdir_parts)]: + if quiet < 2: + print("The stripdir path {!r} is not a valid prefix for " + "source path {!r}; ignoring".format(stripdir, fullname)) + else: + dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) if prependdir is not None: if dfile is None: diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 816edab99f63e3..ffaffdb8b3d0aa 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -71,6 +71,11 @@ def __init__(self): self._reader, self._writer = mp.Pipe(duplex=False) def close(self): + # Please note that we do not take the shutdown lock when + # calling clear() (to avoid deadlocking) so this method can + # only be called safely from the same thread as all calls to + # clear() even if you hold the shutdown lock. Otherwise we + # might try to read from the closed pipe. if not self._closed: self._closed = True self._writer.close() @@ -336,7 +341,14 @@ def run(self): # Main loop for the executor manager thread. while True: - self.add_call_item_to_queue() + # gh-109047: During Python finalization, self.call_queue.put() + # creation of a thread can fail with RuntimeError. + try: + self.add_call_item_to_queue() + except BaseException as exc: + cause = format_exception(exc) + self.terminate_broken(cause) + return result_item, is_broken, cause = self.wait_result_broken_or_wakeup() @@ -420,14 +432,18 @@ def wait_result_broken_or_wakeup(self): try: result_item = result_reader.recv() is_broken = False - except BaseException as e: - cause = format_exception(type(e), e, e.__traceback__) + except BaseException as exc: + cause = format_exception(exc) elif wakeup_reader in ready: is_broken = False - with self.shutdown_lock: - self.thread_wakeup.clear() + # No need to hold the _shutdown_lock here because: + # 1. we're the only thread to use the wakeup reader + # 2. we're also the only thread to call thread_wakeup.close() + # 3. we want to avoid a possible deadlock when both reader and writer + # would block (gh-105829) + self.thread_wakeup.clear() return result_item, is_broken, cause @@ -435,24 +451,14 @@ def process_result_item(self, result_item): # Process the received a result_item. This can be either the PID of a # worker that exited gracefully or a _ResultItem - if isinstance(result_item, int): - # Clean shutdown of a worker using its PID - # (avoids marking the executor broken) - assert self.is_shutting_down() - p = self.processes.pop(result_item) - p.join() - if not self.processes: - self.join_executor_internals() - return - else: - # Received a _ResultItem so mark the future as completed. - work_item = self.pending_work_items.pop(result_item.work_id, None) - # work_item can be None if another process terminated (see above) - if work_item is not None: - if result_item.exception: - work_item.future.set_exception(result_item.exception) - else: - work_item.future.set_result(result_item.result) + # Received a _ResultItem so mark the future as completed. + work_item = self.pending_work_items.pop(result_item.work_id, None) + # work_item can be None if another process terminated (see above) + if work_item is not None: + if result_item.exception: + work_item.future.set_exception(result_item.exception) + else: + work_item.future.set_result(result_item.result) def is_shutting_down(self): # Check whether we should start shutting down the executor. @@ -464,7 +470,7 @@ def is_shutting_down(self): return (_global_shutdown or executor is None or executor._shutdown_thread) - def terminate_broken(self, cause): + def _terminate_broken(self, cause): # Terminate the executor because it is in a broken state. The cause # argument can be used to display more information on the error that # lead the executor into becoming broken. @@ -489,7 +495,14 @@ def terminate_broken(self, cause): # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): - work_item.future.set_exception(bpe) + try: + work_item.future.set_exception(bpe) + except _base.InvalidStateError: + # set_exception() fails if the future is cancelled: ignore it. + # Trying to check if the future is cancelled before calling + # set_exception() would leave a race condition if the future is + # cancelled between the check and set_exception(). + pass # Delete references to object. See issue16284 del work_item self.pending_work_items.clear() @@ -499,8 +512,14 @@ def terminate_broken(self, cause): for p in self.processes.values(): p.terminate() + self.call_queue._terminate_broken() + # clean up resources - self.join_executor_internals() + self._join_executor_internals(broken=True) + + def terminate_broken(self, cause): + with self.shutdown_lock: + self._terminate_broken(cause) def flag_executor_shutting_down(self): # Flag the executor as shutting down and cancel remaining tasks if @@ -543,15 +562,24 @@ def shutdown_workers(self): break def join_executor_internals(self): - self.shutdown_workers() + with self.shutdown_lock: + self._join_executor_internals() + + def _join_executor_internals(self, broken=False): + # If broken, call_queue was closed and so can no longer be used. + if not broken: + self.shutdown_workers() + # Release the queue's resources as soon as possible. self.call_queue.close() self.call_queue.join_thread() - with self.shutdown_lock: - self.thread_wakeup.close() + self.thread_wakeup.close() + # If .join() is not called on the created processes then # some ctx.Queue methods may deadlock on Mac OS X. for p in self.processes.values(): + if broken: + p.terminate() p.join() def get_n_children_alive(self): @@ -638,7 +666,7 @@ def __init__(self, max_workers=None, mp_context=None, _check_system_limits() if max_workers is None: - self._max_workers = os.cpu_count() or 1 + self._max_workers = os.process_cpu_count() or 1 if sys.platform == 'win32': self._max_workers = min(_MAX_WINDOWS_WORKERS, self._max_workers) @@ -702,7 +730,10 @@ def __init__(self, max_workers=None, mp_context=None, # as it could result in a deadlock if a worker process dies with the # _result_queue write lock still acquired. # - # _shutdown_lock must be locked to access _ThreadWakeup. + # _shutdown_lock must be locked to access _ThreadWakeup.close() and + # .wakeup(). Care must also be taken to not call clear or close from + # more than one thread since _ThreadWakeup.clear() is not protected by + # the _shutdown_lock self._executor_manager_thread_wakeup = _ThreadWakeup() # Create communication channels for the executor diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 51c942f51abd37..a024033f35fb54 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -43,7 +43,7 @@ def _python_exit(): after_in_parent=_global_shutdown_lock.release) -class _WorkItem(object): +class _WorkItem: def __init__(self, future, fn, args, kwargs): self.future = future self.fn = fn @@ -78,17 +78,20 @@ def _worker(executor_reference, work_queue, initializer, initargs): return try: while True: - work_item = work_queue.get(block=True) - if work_item is not None: - work_item.run() - # Delete references to object. See issue16284 - del work_item - - # attempt to increment idle count + try: + work_item = work_queue.get_nowait() + except queue.Empty: + # attempt to increment idle count if queue is empty executor = executor_reference() if executor is not None: executor._idle_semaphore.release() del executor + work_item = work_queue.get(block=True) + + if work_item is not None: + work_item.run() + # Delete references to object. See GH-60488 + del work_item continue executor = executor_reference() @@ -136,10 +139,10 @@ def __init__(self, max_workers=None, thread_name_prefix='', # * CPU bound task which releases GIL # * I/O bound task (which releases GIL, of course) # - # We use cpu_count + 4 for both types of tasks. + # We use process_cpu_count + 4 for both types of tasks. # But we limit it to 32 to avoid consuming surprisingly large resource # on many core machine. - max_workers = min(32, (os.cpu_count() or 1) + 4) + max_workers = min(32, (os.process_cpu_count() or 1) + 4) if max_workers <= 0: raise ValueError("max_workers must be greater than 0") diff --git a/Lib/configparser.py b/Lib/configparser.py index dee5a0db7e7ddc..71362d23ec3757 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -59,7 +59,7 @@ instance. It will be used as the handler for option value pre-processing when using getters. RawConfigParser objects don't do any sort of interpolation, whereas ConfigParser uses an instance of - BasicInterpolation. The library also provides a ``zc.buildbot`` + BasicInterpolation. The library also provides a ``zc.buildout`` inspired ExtendedInterpolation implementation. When `converters` is given, it should be a dictionary where each key @@ -147,7 +147,6 @@ import os import re import sys -import warnings __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", @@ -155,7 +154,7 @@ "ParsingError", "MissingSectionHeaderError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", - "LegacyInterpolation", "SectionProxy", "ConverterMapping", + "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH") _default_dict = dict @@ -491,53 +490,6 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, "found: %r" % (rest,)) -class LegacyInterpolation(Interpolation): - """Deprecated interpolation used in old versions of ConfigParser. - Use BasicInterpolation or ExtendedInterpolation instead.""" - - _KEYCRE = re.compile(r"%\(([^)]*)\)s|.") - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - warnings.warn( - "LegacyInterpolation has been deprecated since Python 3.2 " - "and will be removed from the configparser module in Python 3.13. " - "Use BasicInterpolation or ExtendedInterpolation instead.", - DeprecationWarning, stacklevel=2 - ) - - def before_get(self, parser, section, option, value, vars): - rawval = value - depth = MAX_INTERPOLATION_DEPTH - while depth: # Loop through this until it's done - depth -= 1 - if value and "%(" in value: - replace = functools.partial(self._interpolation_replace, - parser=parser) - value = self._KEYCRE.sub(replace, value) - try: - value = value % vars - except KeyError as e: - raise InterpolationMissingOptionError( - option, section, rawval, e.args[0]) from None - else: - break - if value and "%(" in value: - raise InterpolationDepthError(option, section, rawval) - return value - - def before_set(self, parser, section, option, value): - return value - - @staticmethod - def _interpolation_replace(match, parser): - s = match.group(1) - if s is None: - return match.group() - else: - return "%%(%s)s" % parser.optionxform(s) - - class RawConfigParser(MutableMapping): """ConfigParser that does not do interpolation.""" diff --git a/Lib/contextlib.py b/Lib/contextlib.py index b5acbcb9e6d77c..5b646fabca0225 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -20,6 +20,8 @@ class AbstractContextManager(abc.ABC): __class_getitem__ = classmethod(GenericAlias) + __slots__ = () + def __enter__(self): """Return `self` upon entering the runtime context.""" return self @@ -42,6 +44,8 @@ class AbstractAsyncContextManager(abc.ABC): __class_getitem__ = classmethod(GenericAlias) + __slots__ = () + async def __aenter__(self): """Return `self` upon entering the runtime context.""" return self @@ -145,7 +149,10 @@ def __exit__(self, typ, value, traceback): except StopIteration: return False else: - raise RuntimeError("generator didn't stop") + try: + raise RuntimeError("generator didn't stop") + finally: + self.gen.close() else: if value is None: # Need to force instantiation so we can reliably @@ -187,7 +194,10 @@ def __exit__(self, typ, value, traceback): raise exc.__traceback__ = traceback return False - raise RuntimeError("generator didn't stop after throw()") + try: + raise RuntimeError("generator didn't stop after throw()") + finally: + self.gen.close() class _AsyncGeneratorContextManager( _GeneratorContextManagerBase, @@ -212,7 +222,10 @@ async def __aexit__(self, typ, value, traceback): except StopAsyncIteration: return False else: - raise RuntimeError("generator didn't stop") + try: + raise RuntimeError("generator didn't stop") + finally: + await self.gen.aclose() else: if value is None: # Need to force instantiation so we can reliably @@ -254,7 +267,10 @@ async def __aexit__(self, typ, value, traceback): raise exc.__traceback__ = traceback return False - raise RuntimeError("generator didn't stop after athrow()") + try: + raise RuntimeError("generator didn't stop after athrow()") + finally: + await self.gen.aclose() def contextmanager(func): @@ -445,7 +461,7 @@ def __exit__(self, exctype, excinst, exctb): return if issubclass(exctype, self._exceptions): return True - if issubclass(exctype, ExceptionGroup): + if issubclass(exctype, BaseExceptionGroup): match, rest = excinst.split(self._exceptions) if rest is None: return True @@ -553,11 +569,12 @@ def __enter__(self): return self def __exit__(self, *exc_details): - received_exc = exc_details[0] is not None + exc = exc_details[1] + received_exc = exc is not None # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements - frame_exc = sys.exc_info()[1] + frame_exc = sys.exception() def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: @@ -580,24 +597,28 @@ def _fix_exception_context(new_exc, old_exc): is_sync, cb = self._exit_callbacks.pop() assert is_sync try: + if exc is None: + exc_details = None, None, None + else: + exc_details = type(exc), exc, exc.__traceback__ if cb(*exc_details): suppressed_exc = True pending_raise = False - exc_details = (None, None, None) - except: - new_exc_details = sys.exc_info() + exc = None + except BaseException as new_exc: # simulate the stack of exceptions by setting the context - _fix_exception_context(new_exc_details[1], exc_details[1]) + _fix_exception_context(new_exc, exc) pending_raise = True - exc_details = new_exc_details + exc = new_exc + if pending_raise: try: - # bare "raise exc_details[1]" replaces our carefully + # bare "raise exc" replaces our carefully # set-up context - fixed_ctx = exc_details[1].__context__ - raise exc_details[1] + fixed_ctx = exc.__context__ + raise exc except BaseException: - exc_details[1].__context__ = fixed_ctx + exc.__context__ = fixed_ctx raise return received_exc and suppressed_exc @@ -693,11 +714,12 @@ async def __aenter__(self): return self async def __aexit__(self, *exc_details): - received_exc = exc_details[0] is not None + exc = exc_details[1] + received_exc = exc is not None # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements - frame_exc = sys.exc_info()[1] + frame_exc = sys.exception() def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: @@ -719,6 +741,10 @@ def _fix_exception_context(new_exc, old_exc): while self._exit_callbacks: is_sync, cb = self._exit_callbacks.pop() try: + if exc is None: + exc_details = None, None, None + else: + exc_details = type(exc), exc, exc.__traceback__ if is_sync: cb_suppress = cb(*exc_details) else: @@ -727,21 +753,21 @@ def _fix_exception_context(new_exc, old_exc): if cb_suppress: suppressed_exc = True pending_raise = False - exc_details = (None, None, None) - except: - new_exc_details = sys.exc_info() + exc = None + except BaseException as new_exc: # simulate the stack of exceptions by setting the context - _fix_exception_context(new_exc_details[1], exc_details[1]) + _fix_exception_context(new_exc, exc) pending_raise = True - exc_details = new_exc_details + exc = new_exc + if pending_raise: try: - # bare "raise exc_details[1]" replaces our carefully + # bare "raise exc" replaces our carefully # set-up context - fixed_ctx = exc_details[1].__context__ - raise exc_details[1] + fixed_ctx = exc.__context__ + raise exc except BaseException: - exc_details[1].__context__ = fixed_ctx + exc.__context__ = fixed_ctx raise return received_exc and suppressed_exc diff --git a/Lib/copy.py b/Lib/copy.py index da2908ef623d8c..a69bc4e78c20b3 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -121,13 +121,13 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ + d = id(x) if memo is None: memo = {} - - d = id(x) - y = memo.get(d, _nil) - if y is not _nil: - return y + else: + y = memo.get(d, _nil) + if y is not _nil: + return y cls = type(x) @@ -290,3 +290,16 @@ def _reconstruct(x, memo, func, args, return y del types, weakref + + +def replace(obj, /, **changes): + """Return a new object replacing specified fields with new values. + + This is especially useful for immutable objects, like named tuples or + frozen dataclasses. + """ + cls = obj.__class__ + func = getattr(cls, '__replace__', None) + if func is None: + raise TypeError(f"replace() does not support {cls.__name__} objects") + return func(obj, **changes) diff --git a/Lib/crypt.py b/Lib/crypt.py deleted file mode 100644 index de4a14a3884762..00000000000000 --- a/Lib/crypt.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Wrapper to the POSIX crypt library call and associated functionality.""" - -import sys as _sys - -try: - import _crypt -except ModuleNotFoundError: - if _sys.platform == 'win32': - raise ImportError("The crypt module is not supported on Windows") - else: - raise ImportError("The required _crypt module was not built as part of CPython") - -import errno -import string as _string -import warnings -from random import SystemRandom as _SystemRandom -from collections import namedtuple as _namedtuple - - -warnings._deprecated(__name__, remove=(3, 13)) - - -_saltchars = _string.ascii_letters + _string.digits + './' -_sr = _SystemRandom() - - -class _Method(_namedtuple('_Method', 'name ident salt_chars total_size')): - - """Class representing a salt method per the Modular Crypt Format or the - legacy 2-character crypt method.""" - - def __repr__(self): - return ''.format(self.name) - - -def mksalt(method=None, *, rounds=None): - """Generate a salt for the specified method. - - If not specified, the strongest available method will be used. - - """ - if method is None: - method = methods[0] - if rounds is not None and not isinstance(rounds, int): - raise TypeError(f'{rounds.__class__.__name__} object cannot be ' - f'interpreted as an integer') - if not method.ident: # traditional - s = '' - else: # modular - s = f'${method.ident}$' - - if method.ident and method.ident[0] == '2': # Blowfish variants - if rounds is None: - log_rounds = 12 - else: - log_rounds = int.bit_length(rounds-1) - if rounds != 1 << log_rounds: - raise ValueError('rounds must be a power of 2') - if not 4 <= log_rounds <= 31: - raise ValueError('rounds out of the range 2**4 to 2**31') - s += f'{log_rounds:02d}$' - elif method.ident in ('5', '6'): # SHA-2 - if rounds is not None: - if not 1000 <= rounds <= 999_999_999: - raise ValueError('rounds out of the range 1000 to 999_999_999') - s += f'rounds={rounds}$' - elif rounds is not None: - raise ValueError(f"{method} doesn't support the rounds argument") - - s += ''.join(_sr.choice(_saltchars) for char in range(method.salt_chars)) - return s - - -def crypt(word, salt=None): - """Return a string representing the one-way hash of a password, with a salt - prepended. - - If ``salt`` is not specified or is ``None``, the strongest - available method will be selected and a salt generated. Otherwise, - ``salt`` may be one of the ``crypt.METHOD_*`` values, or a string as - returned by ``crypt.mksalt()``. - - """ - if salt is None or isinstance(salt, _Method): - salt = mksalt(salt) - return _crypt.crypt(word, salt) - - -# available salting/crypto methods -methods = [] - -def _add_method(name, *args, rounds=None): - method = _Method(name, *args) - globals()['METHOD_' + name] = method - salt = mksalt(method, rounds=rounds) - result = None - try: - result = crypt('', salt) - except OSError as e: - # Not all libc libraries support all encryption methods. - if e.errno in {errno.EINVAL, errno.EPERM, errno.ENOSYS}: - return False - raise - if result and len(result) == method.total_size: - methods.append(method) - return True - return False - -_add_method('SHA512', '6', 16, 106) -_add_method('SHA256', '5', 16, 63) - -# Choose the strongest supported version of Blowfish hashing. -# Early versions have flaws. Version 'a' fixes flaws of -# the initial implementation, 'b' fixes flaws of 'a'. -# 'y' is the same as 'b', for compatibility -# with openwall crypt_blowfish. -for _v in 'b', 'y', 'a', '': - if _add_method('BLOWFISH', '2' + _v, 22, 59 + len(_v), rounds=1<<4): - break - -_add_method('MD5', '1', 8, 34) -_add_method('CRYPT', None, 2, 13) - -del _v, _add_method diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 95353bab26cc71..141142a57dcb3e 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -302,8 +302,9 @@ def create_unicode_buffer(init, size=None): raise TypeError(init) -# XXX Deprecated def SetPointerType(pointer, cls): + import warnings + warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) if _pointer_type_cache.get(cls, None) is not None: raise RuntimeError("This type already exists in the cache") if id(pointer) not in _pointer_type_cache: @@ -312,8 +313,9 @@ def SetPointerType(pointer, cls): _pointer_type_cache[cls] = pointer del _pointer_type_cache[id(pointer)] -# XXX Deprecated def ARRAY(typ, len): + import warnings + warnings._deprecated("ctypes.ARRAY", remove=(3, 15)) return typ * len ################################################################ diff --git a/Lib/ctypes/_endian.py b/Lib/ctypes/_endian.py index b5446c049bc9dc..3febb3118b8230 100644 --- a/Lib/ctypes/_endian.py +++ b/Lib/ctypes/_endian.py @@ -1,5 +1,5 @@ import sys -from ctypes import * +from ctypes import Array, Structure, Union _array_type = type(Array) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3eacba840db426..2fba32b5ffbc1e 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -575,15 +575,15 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, # message, and future-proofs us in case we build up the function # using ast. - seen_default = False + seen_default = None for f in std_fields: # Only consider the non-kw-only fields in the __init__ call. if f.init: if not (f.default is MISSING and f.default_factory is MISSING): - seen_default = True + seen_default = f elif seen_default: raise TypeError(f'non-default argument {f.name!r} ' - 'follows default argument') + f'follows default argument {seen_default.name!r}') locals = {f'__dataclass_type_{f.name}__': f.type for f in fields} locals.update({ @@ -627,7 +627,7 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, def _repr_fn(fields, globals): fn = _create_fn('__repr__', ('self',), - ['return self.__class__.__qualname__ + f"(' + + ['return f"{self.__class__.__qualname__}(' + ', '.join([f"{f.name}={{self.{f.name}!r}}" for f in fields]) + ')"'], @@ -944,8 +944,11 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # Find our base classes in reverse MRO order, and exclude # ourselves. In reversed order so that more derived classes # override earlier field definitions in base classes. As long as - # we're iterating over them, see if any are frozen. + # we're iterating over them, see if all or any of them are frozen. any_frozen_base = False + # By default `all_frozen_bases` is `None` to represent a case, + # where some dataclasses does not have any bases with `_FIELDS` + all_frozen_bases = None has_dataclass_bases = False for b in cls.__mro__[-1:0:-1]: # Only process classes that have been processed by our @@ -955,8 +958,11 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f - if getattr(b, _PARAMS).frozen: - any_frozen_base = True + if all_frozen_bases is None: + all_frozen_bases = True + current_frozen = getattr(b, _PARAMS).frozen + all_frozen_bases = all_frozen_bases and current_frozen + any_frozen_base = any_frozen_base or current_frozen # Annotations defined specifically in this class (not in base classes). # @@ -1025,7 +1031,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, 'frozen one') # Raise an exception if we're frozen, but none of our bases are. - if not any_frozen_base and frozen: + if all_frozen_bases is False and frozen: raise TypeError('cannot inherit frozen dataclass from a ' 'non-frozen one') @@ -1036,7 +1042,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically # set __hash__ to None. This is a heuristic, as it's possible - # that such a __hash__ == None was not auto-generated, but it + # that such a __hash__ == None was not auto-generated, but it's # close enough. class_hash = cls.__dict__.get('__hash__', MISSING) has_explicit_hash = not (class_hash is MISSING or @@ -1073,6 +1079,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, globals, slots, )) + _set_new_attribute(cls, '__replace__', _replace) # Get the fields as a list, and include only real fields. This is # used in all of the following methods. @@ -1085,13 +1092,17 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if eq: # Create __eq__ method. There's no need for a __ne__ method, # since python will call __eq__ and negate it. - flds = [f for f in field_list if f.compare] - self_tuple = _tuple_str('self', flds) - other_tuple = _tuple_str('other', flds) - _set_new_attribute(cls, '__eq__', - _cmp_fn('__eq__', '==', - self_tuple, other_tuple, - globals=globals)) + cmp_fields = (field for field in field_list if field.compare) + terms = [f'self.{field.name}==other.{field.name}' for field in cmp_fields] + field_comparisons = ' and '.join(terms) or 'True' + body = [f'if other.__class__ is self.__class__:', + f' return {field_comparisons}', + f'return NotImplemented'] + func = _create_fn('__eq__', + ('self', 'other'), + body, + globals=globals) + _set_new_attribute(cls, '__eq__', func) if order: # Create and set the ordering methods. @@ -1542,13 +1553,15 @@ class C: c1 = replace(c, x=3) assert c1.x == 3 and c1.y == 2 """ + if not _is_dataclass_instance(obj): + raise TypeError("replace() should be called on dataclass instances") + return _replace(obj, **changes) + +def _replace(obj, /, **changes): # We're going to mutate 'changes', but that's okay because it's a # new dict, even if called with 'replace(obj, **my_changes)'. - if not _is_dataclass_instance(obj): - raise TypeError("replace() should be called on dataclass instances") - # It's an error to have init=False fields in 'changes'. # If a field is not in 'changes', read its value from the provided obj. @@ -1560,15 +1573,15 @@ class C: if not f.init: # Error if this field is specified in changes. if f.name in changes: - raise ValueError(f'field {f.name} is declared with ' - 'init=False, it cannot be specified with ' - 'replace()') + raise TypeError(f'field {f.name} is declared with ' + f'init=False, it cannot be specified with ' + f'replace()') continue if f.name not in changes: if f._field_type is _FIELD_INITVAR and f.default is MISSING: - raise ValueError(f"InitVar {f.name!r} " - 'must be specified with replace()') + raise TypeError(f"InitVar {f.name!r} " + f'must be specified with replace()') changes[f.name] = getattr(obj, f.name) # Create the new object, which calls __init__() and diff --git a/Lib/datetime.py b/Lib/datetime.py index bad8beb4f6b026..a33d2d724cb33d 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -6,4 +6,4 @@ from _pydatetime import __doc__ __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", - "MINYEAR", "MAXYEAR") + "MINYEAR", "MAXYEAR", "UTC") diff --git a/Lib/dis.py b/Lib/dis.py index 3a8e6ac3bf5ace..efa935c5a6a0b6 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -14,7 +14,7 @@ _intrinsic_1_descs, _intrinsic_2_descs, _specializations, - _specialized_instructions, + _specialized_opmap, ) __all__ = ["code_info", "dis", "disassemble", "distb", "disco", @@ -25,16 +25,12 @@ _have_code = (types.MethodType, types.FunctionType, types.CodeType, classmethod, staticmethod, type) -FORMAT_VALUE = opmap['FORMAT_VALUE'] -FORMAT_VALUE_CONVERTERS = ( - (None, ''), - (str, 'str'), - (repr, 'repr'), - (ascii, 'ascii'), -) -MAKE_FUNCTION = opmap['MAKE_FUNCTION'] -MAKE_FUNCTION_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') +CONVERT_VALUE = opmap['CONVERT_VALUE'] + +SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] +FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') +ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] LOAD_CONST = opmap['LOAD_CONST'] RETURN_CONST = opmap['RETURN_CONST'] LOAD_GLOBAL = opmap['LOAD_GLOBAL'] @@ -46,16 +42,19 @@ LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] +LOAD_FAST_LOAD_FAST = opmap['LOAD_FAST_LOAD_FAST'] +STORE_FAST_LOAD_FAST = opmap['STORE_FAST_LOAD_FAST'] +STORE_FAST_STORE_FAST = opmap['STORE_FAST_STORE_FAST'] CACHE = opmap["CACHE"] _all_opname = list(opname) _all_opmap = dict(opmap) -_empty_slot = [slot for slot, name in enumerate(_all_opname) if name.startswith("<")] -for spec_op, specialized in zip(_empty_slot, _specialized_instructions): +for name, op in _specialized_opmap.items(): # fill opname and opmap - _all_opname[spec_op] = specialized - _all_opmap[specialized] = spec_op + assert op < len(_all_opname) + _all_opname[op] = name + _all_opmap[name] = op deoptmap = { specialized: base for base, family in _specializations.items() for specialized in family @@ -74,7 +73,8 @@ def _try_compile(source, name): pass return compile(source, name, 'exec') -def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False): +def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, + show_offsets=False): """Disassemble classes, methods, functions, and other compiled objects. With no argument, disassemble the last traceback. @@ -84,7 +84,8 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False): in a special attribute. """ if x is None: - distb(file=file, show_caches=show_caches, adaptive=adaptive) + distb(file=file, show_caches=show_caches, adaptive=adaptive, + show_offsets=show_offsets) return # Extract functions from methods. if hasattr(x, '__func__'): @@ -105,21 +106,21 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False): if isinstance(x1, _have_code): print("Disassembly of %s:" % name, file=file) try: - dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive) + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) except TypeError as msg: print("Sorry:", msg, file=file) print(file=file) elif hasattr(x, 'co_code'): # Code object - _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive) + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) elif isinstance(x, (bytes, bytearray)): # Raw bytecode - _disassemble_bytes(x, file=file, show_caches=show_caches) + _disassemble_bytes(x, file=file, show_caches=show_caches, show_offsets=show_offsets) elif isinstance(x, str): # Source code - _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive) + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) -def distb(tb=None, *, file=None, show_caches=False, adaptive=False): +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False): """Disassemble a traceback (default: last traceback).""" if tb is None: try: @@ -130,7 +131,7 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False): except AttributeError: raise RuntimeError("no last traceback to disassemble") from None while tb.tb_next: tb = tb.tb_next - disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive) + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to @@ -262,11 +263,13 @@ def show_code(co, *, file=None): 'argval', 'argrepr', 'offset', + 'start_offset', 'starts_line', - 'is_jump_target', + 'line_number', + 'label', 'positions' ], - defaults=[None] + defaults=[None, None] ) _Instruction.opname.__doc__ = "Human readable name for operation" @@ -275,18 +278,47 @@ def show_code(co, *, file=None): _Instruction.argval.__doc__ = "Resolved arg value (if known), otherwise same as arg" _Instruction.argrepr.__doc__ = "Human readable description of operation argument" _Instruction.offset.__doc__ = "Start index of operation within bytecode sequence" -_Instruction.starts_line.__doc__ = "Line started by this opcode (if any), otherwise None" -_Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False" +_Instruction.start_offset.__doc__ = ( + "Start index of operation within bytecode sequence, including extended args if present; " + "otherwise equal to Instruction.offset" +) +_Instruction.starts_line.__doc__ = "True if this opcode starts a source line, otherwise False" +_Instruction.line_number.__doc__ = "source line number associated with this opcode (if any), otherwise None" +_Instruction.label.__doc__ = "A label (int > 0) if this instruction is a jump target, otherwise None" _Instruction.positions.__doc__ = "dis.Positions object holding the span of source code covered by this instruction" -_ExceptionTableEntry = collections.namedtuple("_ExceptionTableEntry", +_ExceptionTableEntryBase = collections.namedtuple("_ExceptionTableEntryBase", "start end target depth lasti") +class _ExceptionTableEntry(_ExceptionTableEntryBase): + pass + _OPNAME_WIDTH = 20 _OPARG_WIDTH = 5 +def _get_cache_size(opname): + return _inline_cache_entries.get(opname, 0) + +def _get_jump_target(op, arg, offset): + """Gets the bytecode offset of the jump target if this is a jump instruction. + + Otherwise return None. + """ + deop = _deoptop(op) + caches = _get_cache_size(_all_opname[deop]) + if deop in hasjrel: + if _is_backward_jump(deop): + arg = -arg + target = offset + 2 + arg*2 + target += 2 * caches + elif deop in hasjabs: + target = arg*2 + else: + target = None + return target + class Instruction(_Instruction): - """Details for a bytecode operation + """Details for a bytecode operation. Defined fields: opname - human readable name for operation @@ -295,48 +327,221 @@ class Instruction(_Instruction): argval - resolved arg value (if known), otherwise same as arg argrepr - human readable description of operation argument offset - start index of operation within bytecode sequence - starts_line - line started by this opcode (if any), otherwise None - is_jump_target - True if other code jumps to here, otherwise False + start_offset - start index of operation within bytecode sequence including extended args if present; + otherwise equal to Instruction.offset + starts_line - True if this opcode starts a source line, otherwise False + line_number - source line number associated with this opcode (if any), otherwise None + label - A label if this instruction is a jump target, otherwise None positions - Optional dis.Positions object holding the span of source code covered by this instruction """ - def _disassemble(self, lineno_width=3, mark_as_current=False, offset_width=4): - """Format instruction details for inclusion in disassembly output + @property + def oparg(self): + """Alias for Instruction.arg.""" + return self.arg + + @property + def baseopcode(self): + """Numeric code for the base operation if operation is specialized. + + Otherwise equal to Instruction.opcode. + """ + return _deoptop(self.opcode) + + @property + def baseopname(self): + """Human readable name for the base operation if operation is specialized. + + Otherwise equal to Instruction.opname. + """ + return opname[self.baseopcode] + + @property + def cache_offset(self): + """Start index of the cache entries following the operation.""" + return self.offset + 2 + + @property + def end_offset(self): + """End index of the cache entries following the operation.""" + return self.cache_offset + _get_cache_size(_all_opname[self.opcode])*2 + + @property + def jump_target(self): + """Bytecode index of the jump target if this is a jump operation. + Otherwise return None. + """ + return _get_jump_target(self.opcode, self.arg, self.offset) + + @property + def is_jump_target(self): + """True if other code jumps to here, otherwise False""" + return self.label is not None + + def __str__(self): + output = io.StringIO() + formatter = Formatter(file=output) + formatter.print_instruction(self, False) + return output.getvalue() + + +class Formatter: + + def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, + line_offset=0): + """Create a Formatter + + *file* where to write the output *lineno_width* sets the width of the line number field (0 omits it) - *mark_as_current* inserts a '-->' marker arrow as part of the line *offset_width* sets the width of the instruction offset field + *label_width* sets the width of the label field + + *line_offset* the line number (within the code unit) """ + self.file = file + self.lineno_width = lineno_width + self.offset_width = offset_width + self.label_width = label_width + + + def print_instruction(self, instr, mark_as_current=False): + """Format instruction details for inclusion in disassembly output.""" + lineno_width = self.lineno_width + offset_width = self.offset_width + label_width = self.label_width + + new_source_line = (lineno_width > 0 and + instr.starts_line and + instr.offset > 0) + if new_source_line: + print(file=self.file) + fields = [] # Column: Source code line number if lineno_width: - if self.starts_line is not None: - lineno_fmt = "%%%dd" % lineno_width - fields.append(lineno_fmt % self.starts_line) + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" + lineno_fmt = lineno_fmt % lineno_width + lineno = _NO_LINENO if instr.line_number is None else instr.line_number + fields.append(lineno_fmt % lineno) else: fields.append(' ' * lineno_width) + # Column: Label + if instr.label is not None: + lbl = f"L{instr.label}:" + fields.append(f"{lbl:>{label_width}}") + else: + fields.append(' ' * label_width) + # Column: Instruction offset from start of code sequence + if offset_width > 0: + fields.append(f"{repr(instr.offset):>{offset_width}} ") # Column: Current instruction indicator if mark_as_current: fields.append('-->') else: fields.append(' ') - # Column: Jump target marker - if self.is_jump_target: - fields.append('>>') - else: - fields.append(' ') - # Column: Instruction offset from start of code sequence - fields.append(repr(self.offset).rjust(offset_width)) # Column: Opcode name - fields.append(self.opname.ljust(_OPNAME_WIDTH)) + fields.append(instr.opname.ljust(_OPNAME_WIDTH)) # Column: Opcode argument - if self.arg is not None: - fields.append(repr(self.arg).rjust(_OPARG_WIDTH)) + if instr.arg is not None: + arg = repr(instr.arg) + # If opname is longer than _OPNAME_WIDTH, we allow it to overflow into + # the space reserved for oparg. This results in fewer misaligned opargs + # in the disassembly output. + opname_excess = max(0, len(instr.opname) - _OPNAME_WIDTH) + fields.append(repr(instr.arg).rjust(_OPARG_WIDTH - opname_excess)) # Column: Opcode argument details - if self.argrepr: - fields.append('(' + self.argrepr + ')') - return ' '.join(fields).rstrip() + if instr.argrepr: + fields.append('(' + instr.argrepr + ')') + print(' '.join(fields).rstrip(), file=self.file) + + def print_exception_table(self, exception_entries): + file = self.file + if exception_entries: + print("ExceptionTable:", file=file) + for entry in exception_entries: + lasti = " lasti" if entry.lasti else "" + start = entry.start_label + end = entry.end_label + target = entry.target_label + print(f" L{start} to L{end} -> L{target} [{entry.depth}]{lasti}", file=file) + + +class ArgResolver: + def __init__(self, co_consts, names, varname_from_oparg, labels_map): + self.co_consts = co_consts + self.names = names + self.varname_from_oparg = varname_from_oparg + self.labels_map = labels_map + + def get_argval_argrepr(self, op, arg, offset): + get_name = None if self.names is None else self.names.__getitem__ + argval = None + argrepr = '' + deop = _deoptop(op) + if arg is not None: + # Set argval to the dereferenced value of the argument when + # available, and argrepr to the string representation of argval. + # _disassemble_bytes needs the string repr of the + # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. + argval = arg + if deop in hasconst: + argval, argrepr = _get_const_info(deop, arg, self.co_consts) + elif deop in hasname: + if deop == LOAD_GLOBAL: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL" + elif deop == LOAD_ATTR: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + elif deop == LOAD_SUPER_ATTR: + argval, argrepr = _get_name_info(arg//4, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + else: + argval, argrepr = _get_name_info(arg, get_name) + elif deop in hasjabs: + argval = arg*2 + argrepr = f"to L{self.labels_map[argval]}" + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + if deop == ENTER_EXECUTOR: + argval += 2 + argrepr = f"to L{self.labels_map[argval]}" + elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): + arg1 = arg >> 4 + arg2 = arg & 15 + val1, argrepr1 = _get_name_info(arg1, self.varname_from_oparg) + val2, argrepr2 = _get_name_info(arg2, self.varname_from_oparg) + argrepr = argrepr1 + ", " + argrepr2 + argval = val1, val2 + elif deop in haslocal or deop in hasfree: + argval, argrepr = _get_name_info(arg, self.varname_from_oparg) + elif deop in hascompare: + argval = cmp_op[arg >> 5] + argrepr = argval + if arg & 16: + argrepr = f"bool({argrepr})" + elif deop == CONVERT_VALUE: + argval = (None, str, repr, ascii)[arg] + argrepr = ('', 'str', 'repr', 'ascii')[arg] + elif deop == SET_FUNCTION_ATTRIBUTE: + argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) + if arg & (1<>4] - argrepr = argval - elif deop == FORMAT_VALUE: - argval, argrepr = FORMAT_VALUE_CONVERTERS[arg & 0x3] - argval = (argval, bool(arg & 0x4)) - if argval[1]: - if argrepr: - argrepr += ', ' - argrepr += 'with format' - elif deop == MAKE_FUNCTION: - argrepr = ', '.join(s for i, s in enumerate(MAKE_FUNCTION_FLAGS) - if arg & (1< 0: if depth is not None: depth = depth - 1 @@ -560,53 +731,81 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adap print(file=file) print("Disassembly of %r:" % (x,), file=file) _disassemble_recursive( - x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive + x, file=file, depth=depth, show_caches=show_caches, + adaptive=adaptive, show_offsets=show_offsets ) + +def _make_labels_map(original_code, exception_entries=()): + jump_targets = set(findlabels(original_code)) + labels = set(jump_targets) + for start, end, target, _, _ in exception_entries: + labels.add(start) + labels.add(end) + labels.add(target) + labels = sorted(labels) + labels_map = {offset: i+1 for (i, offset) in enumerate(sorted(labels))} + for e in exception_entries: + e.start_label = labels_map[e.start] + e.end_label = labels_map[e.end] + e.target_label = labels_map[e.target] + return labels_map + +_NO_LINENO = ' --' + +def _get_lineno_width(linestarts): + if linestarts is None: + return 0 + maxlineno = max(filter(None, linestarts.values()), default=-1) + if maxlineno == -1: + # Omit the line number column entirely if we have no line number info + return 0 + lineno_width = max(3, len(str(maxlineno))) + if lineno_width < len(_NO_LINENO) and None in linestarts.values(): + lineno_width = len(_NO_LINENO) + return lineno_width + + def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None, names=None, co_consts=None, linestarts=None, *, file=None, line_offset=0, exception_entries=(), - co_positions=None, show_caches=False): - # Omit the line number column entirely if we have no line number info - show_lineno = bool(linestarts) - if show_lineno: - maxlineno = max(linestarts.values()) + line_offset - if maxlineno >= 1000: - lineno_width = len(str(maxlineno)) - else: - lineno_width = 3 - else: - lineno_width = 0 - maxoffset = len(code) - 2 - if maxoffset >= 10000: - offset_width = len(str(maxoffset)) - else: - offset_width = 4 - for instr in _get_instructions_bytes(code, varname_from_oparg, names, - co_consts, linestarts, - line_offset=line_offset, - exception_entries=exception_entries, - co_positions=co_positions, - show_caches=show_caches): - new_source_line = (show_lineno and - instr.starts_line is not None and - instr.offset > 0) - if new_source_line: - print(file=file) + co_positions=None, show_caches=False, original_code=None, + show_offsets=False): + + offset_width = len(str(max(len(code) - 2, 9999))) if show_offsets else 0 + + labels_map = _make_labels_map(original_code or code, exception_entries) + label_width = 4 + len(str(len(labels_map))) + + formatter = Formatter(file=file, + lineno_width=_get_lineno_width(linestarts), + offset_width=offset_width, + label_width=label_width, + line_offset=line_offset) + + arg_resolver = ArgResolver(co_consts, names, varname_from_oparg, labels_map) + instrs = _get_instructions_bytes(code, linestarts=linestarts, + line_offset=line_offset, + co_positions=co_positions, + show_caches=show_caches, + original_code=original_code, + labels_map=labels_map, + arg_resolver=arg_resolver) + + print_instructions(instrs, exception_entries, formatter, + show_caches=show_caches, lasti=lasti) + + +def print_instructions(instrs, exception_entries, formatter, show_caches=False, lasti=-1): + for instr in instrs: if show_caches: is_current_instr = instr.offset == lasti else: # Each CACHE takes 2 bytes is_current_instr = instr.offset <= lasti \ - <= instr.offset + 2 * _inline_cache_entries[_deoptop(instr.opcode)] - print(instr._disassemble(lineno_width, is_current_instr, offset_width), - file=file) - if exception_entries: - print("ExceptionTable:", file=file) - for entry in exception_entries: - lasti = " lasti" if entry.lasti else "" - end = entry.end-2 - print(f" {entry.start} to {end} -> {entry.target} [{entry.depth}]{lasti}", file=file) + <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) + formatter.print_instruction(instr, is_current_instr) + formatter.print_exception_table(exception_entries) def _disassemble_str(source, **kwargs): """Compile the source string, then disassemble the code object.""" @@ -622,6 +821,7 @@ def _disassemble_str(source, **kwargs): def _unpack_opargs(code): extended_arg = 0 + extended_args_offset = 0 # Number of EXTENDED_ARG instructions preceding the current instruction caches = 0 for i in range(0, len(code), 2): # Skip inline CACHE entries: @@ -630,7 +830,7 @@ def _unpack_opargs(code): continue op = code[i] deop = _deoptop(op) - caches = _inline_cache_entries[deop] + caches = _get_cache_size(_all_opname[deop]) if deop in hasarg: arg = code[i+1] | extended_arg extended_arg = (arg << 8) if deop == EXTENDED_ARG else 0 @@ -642,7 +842,13 @@ def _unpack_opargs(code): else: arg = None extended_arg = 0 - yield (i, op, arg) + if deop == EXTENDED_ARG: + extended_args_offset += 1 + yield (i, i, op, arg) + else: + start_offset = i - extended_args_offset*2 + yield (i, start_offset, op, arg) + extended_args_offset = 0 def findlabels(code): """Detect all offsets in a byte code which are jump targets. @@ -651,18 +857,10 @@ def findlabels(code): """ labels = [] - for offset, op, arg in _unpack_opargs(code): + for offset, _, op, arg in _unpack_opargs(code): if arg is not None: - deop = _deoptop(op) - caches = _inline_cache_entries[deop] - if deop in hasjrel: - if _is_backward_jump(deop): - arg = -arg - label = offset + 2 + arg*2 - label += 2 * caches - elif deop in hasjabs: - label = arg*2 - else: + label = _get_jump_target(op, arg, offset) + if label is None: continue if label not in labels: labels.append(label) @@ -672,10 +870,12 @@ def findlinestarts(code): """Find the offsets in a byte code which are start of lines in the source. Generate pairs (offset, lineno) + lineno will be an integer or None the offset does not have a source line. """ - lastline = None + + lastline = False # None is a valid line number for start, end, line in code.co_lines(): - if line is not None and line != lastline: + if line is not lastline: lastline = line yield start, line return @@ -691,7 +891,7 @@ def _find_imports(co): consts = co.co_consts names = co.co_names - opargs = [(op, arg) for _, op, arg in _unpack_opargs(co.co_code) + opargs = [(op, arg) for _, _, op, arg in _unpack_opargs(co.co_code) if op != EXTENDED_ARG] for i, (op, oparg) in enumerate(opargs): if op == IMPORT_NAME and i >= 2: @@ -713,7 +913,7 @@ def _find_store_names(co): } names = co.co_names - for _, op, arg in _unpack_opargs(co.co_code): + for _, _, op, arg in _unpack_opargs(co.co_code): if op in STORE_OPS: yield names[arg] @@ -726,7 +926,7 @@ class Bytecode: Iterating over this yields the bytecode operations as Instruction instances. """ - def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False): + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False): self.codeobj = co = _get_code_object(x) if first_line is None: self.first_line = co.co_firstlineno @@ -740,17 +940,22 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False self.exception_entries = _parse_exception_table(co) self.show_caches = show_caches self.adaptive = adaptive + self.show_offsets = show_offsets def __iter__(self): co = self.codeobj + original_code = co.co_code + labels_map = _make_labels_map(original_code, self.exception_entries) + arg_resolver = ArgResolver(co.co_consts, co.co_names, co._varname_from_oparg, + labels_map) return _get_instructions_bytes(_get_code_array(co, self.adaptive), - co._varname_from_oparg, - co.co_names, co.co_consts, - self._linestarts, + linestarts=self._linestarts, line_offset=self._line_offset, - exception_entries=self.exception_entries, co_positions=co.co_positions(), - show_caches=self.show_caches) + show_caches=self.show_caches, + original_code=original_code, + labels_map=labels_map, + arg_resolver=arg_resolver) def __repr__(self): return "{}({!r})".format(self.__class__.__name__, @@ -786,21 +991,26 @@ def dis(self): lasti=offset, exception_entries=self.exception_entries, co_positions=co.co_positions(), - show_caches=self.show_caches) + show_caches=self.show_caches, + original_code=co.co_code, + show_offsets=self.show_offsets) return output.getvalue() -def _test(): - """Simple test program to disassemble a file.""" +def main(): import argparse parser = argparse.ArgumentParser() + parser.add_argument('-C', '--show-caches', action='store_true', + help='show inline caches') + parser.add_argument('-O', '--show-offsets', action='store_true', + help='show instruction offsets') parser.add_argument('infile', type=argparse.FileType('rb'), nargs='?', default='-') args = parser.parse_args() with args.infile as infile: source = infile.read() code = compile(source, args.infile.name, "exec") - dis(code) + dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets) if __name__ == "__main__": - _test() + main() diff --git a/Lib/doctest.py b/Lib/doctest.py index 2776d74bf9b586..d109b6c9e37343 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -105,7 +105,23 @@ def _test(): from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple -TestResults = namedtuple('TestResults', 'failed attempted') + +class TestResults(namedtuple('TestResults', 'failed attempted')): + def __new__(cls, failed, attempted, *, skipped=0): + results = super().__new__(cls, failed, attempted) + results.skipped = skipped + return results + + def __repr__(self): + if self.skipped: + return (f'TestResults(failed={self.failed}, ' + f'attempted={self.attempted}, ' + f'skipped={self.skipped})') + else: + # Leave the repr() unchanged for backward compatibility + # if skipped is zero + return super().__repr__() + # There are 4 basic classes: # - Example: a pair, plus an intra-docstring line number. @@ -575,9 +591,11 @@ def __hash__(self): def __lt__(self, other): if not isinstance(other, DocTest): return NotImplemented - return ((self.name, self.filename, self.lineno, id(self)) + self_lno = self.lineno if self.lineno is not None else -1 + other_lno = other.lineno if other.lineno is not None else -1 + return ((self.name, self.filename, self_lno, id(self)) < - (other.name, other.filename, other.lineno, id(other))) + (other.name, other.filename, other_lno, id(other))) ###################################################################### ## 3. DocTestParser @@ -1110,7 +1128,7 @@ def _find_lineno(self, obj, source_lines): if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) + re.escape(getattr(obj, '__name__', '-'))) for i, line in enumerate(source_lines): if pat.match(line): lineno = i @@ -1150,8 +1168,7 @@ class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. The `run` method is used to process a single DocTest case. It - returns a tuple `(f, t)`, where `t` is the number of test cases - tried, and `f` is the number of test cases that failed. + returns a TestResults instance. >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) @@ -1164,8 +1181,8 @@ class DocTestRunner: _TestClass.square -> TestResults(failed=0, attempted=1) The `summarize` method prints a summary of all the test cases that - have been run by the runner, and returns an aggregated `(f, t)` - tuple: + have been run by the runner, and returns an aggregated TestResults + instance: >>> runner.summarize(verbose=1) 4 items passed all tests: @@ -1178,13 +1195,15 @@ class DocTestRunner: Test passed. TestResults(failed=0, attempted=7) - The aggregated number of tried examples and failed examples is - also available via the `tries` and `failures` attributes: + The aggregated number of tried examples and failed examples is also + available via the `tries`, `failures` and `skips` attributes: >>> runner.tries 7 >>> runner.failures 0 + >>> runner.skips + 0 The comparison between expected outputs and actual outputs is done by an `OutputChecker`. This comparison may be customized with a @@ -1233,7 +1252,8 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Keep track of the examples we've run. self.tries = 0 self.failures = 0 - self._name2ft = {} + self.skips = 0 + self._stats = {} # Create a fake output target for capturing doctest output. self._fakeout = _SpoofOut() @@ -1302,13 +1322,11 @@ def __run(self, test, compileflags, out): Run the examples in `test`. Write the outcome of each example with one of the `DocTestRunner.report_*` methods, using the writer function `out`. `compileflags` is the set of compiler - flags that should be used to execute examples. Return a tuple - `(f, t)`, where `t` is the number of examples tried, and `f` - is the number of examples that failed. The examples are run - in the namespace `test.globs`. + flags that should be used to execute examples. Return a TestResults + instance. The examples are run in the namespace `test.globs`. """ - # Keep track of the number of failures and tries. - failures = tries = 0 + # Keep track of the number of failed, attempted, skipped examples. + failures = attempted = skips = 0 # Save the option flags (since option directives can be used # to modify them). @@ -1320,6 +1338,7 @@ def __run(self, test, compileflags, out): # Process each example. for examplenum, example in enumerate(test.examples): + attempted += 1 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. @@ -1337,10 +1356,10 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + skips += 1 continue # Record that we started this example. - tries += 1 if not quiet: self.report_start(out, test, example) @@ -1376,7 +1395,24 @@ def __run(self, test, compileflags, out): # The example raised an exception: check if it was expected. else: - exc_msg = traceback.format_exception_only(*exception[:2])[-1] + formatted_ex = traceback.format_exception_only(*exception[:2]) + if issubclass(exception[0], SyntaxError): + # SyntaxError / IndentationError is special: + # we don't care about the carets / suggestions / etc + # We only care about the error message and notes. + # They start with `SyntaxError:` (or any other class name) + exception_line_prefixes = ( + f"{exception[0].__qualname__}:", + f"{exception[0].__module__}.{exception[0].__qualname__}:", + ) + exc_msg_index = next( + index + for index, line in enumerate(formatted_ex) + if line.startswith(exception_line_prefixes) + ) + formatted_ex = formatted_ex[exc_msg_index:] + + exc_msg = "".join(formatted_ex) if not quiet: got += _exception_traceback(exception) @@ -1418,19 +1454,22 @@ def __run(self, test, compileflags, out): # Restore the option flags (in case they were modified) self.optionflags = original_optionflags - # Record and return the number of failures and tries. - self.__record_outcome(test, failures, tries) - return TestResults(failures, tries) + # Record and return the number of failures and attempted. + self.__record_outcome(test, failures, attempted, skips) + return TestResults(failures, attempted, skipped=skips) - def __record_outcome(self, test, f, t): + def __record_outcome(self, test, failures, tries, skips): """ - Record the fact that the given DocTest (`test`) generated `f` - failures out of `t` tried examples. + Record the fact that the given DocTest (`test`) generated `failures` + failures out of `tries` tried examples. """ - f2, t2 = self._name2ft.get(test.name, (0,0)) - self._name2ft[test.name] = (f+f2, t+t2) - self.failures += f - self.tries += t + failures2, tries2, skips2 = self._stats.get(test.name, (0, 0, 0)) + self._stats[test.name] = (failures + failures2, + tries + tries2, + skips + skips2) + self.failures += failures + self.tries += tries + self.skips += skips __LINECACHE_FILENAME_RE = re.compile(r'.+)' @@ -1519,9 +1558,7 @@ def out(s): def summarize(self, verbose=None): """ Print a summary of all the test cases that have been run by - this DocTestRunner, and return a tuple `(f, t)`, where `f` is - the total number of failed examples, and `t` is the total - number of tried examples. + this DocTestRunner, and return a TestResults instance. The optional `verbose` argument controls how detailed the summary is. If the verbosity is not specified, then the @@ -1532,59 +1569,61 @@ def summarize(self, verbose=None): notests = [] passed = [] failed = [] - totalt = totalf = 0 - for x in self._name2ft.items(): - name, (f, t) = x - assert f <= t - totalt += t - totalf += f - if t == 0: + total_tries = total_failures = total_skips = 0 + for item in self._stats.items(): + name, (failures, tries, skips) = item + assert failures <= tries + total_tries += tries + total_failures += failures + total_skips += skips + if tries == 0: notests.append(name) - elif f == 0: - passed.append( (name, t) ) + elif failures == 0: + passed.append((name, tries)) else: - failed.append(x) + failed.append(item) if verbose: if notests: - print(len(notests), "items had no tests:") + print(f"{len(notests)} items had no tests:") notests.sort() - for thing in notests: - print(" ", thing) + for name in notests: + print(f" {name}") if passed: - print(len(passed), "items passed all tests:") + print(f"{len(passed)} items passed all tests:") passed.sort() - for thing, count in passed: - print(" %3d tests in %s" % (count, thing)) + for name, count in passed: + print(f" {count:3d} tests in {name}") if failed: print(self.DIVIDER) - print(len(failed), "items had failures:") + print(f"{len(failed)} items had failures:") failed.sort() - for thing, (f, t) in failed: - print(" %3d of %3d in %s" % (f, t, thing)) + for name, (failures, tries, skips) in failed: + print(f" {failures:3d} of {tries:3d} in {name}") if verbose: - print(totalt, "tests in", len(self._name2ft), "items.") - print(totalt - totalf, "passed and", totalf, "failed.") - if totalf: - print("***Test Failed***", totalf, "failures.") + print(f"{total_tries} tests in {len(self._stats)} items.") + print(f"{total_tries - total_failures} passed and {total_failures} failed.") + if total_failures: + msg = f"***Test Failed*** {total_failures} failures" + if total_skips: + msg = f"{msg} and {total_skips} skipped tests" + print(f"{msg}.") elif verbose: print("Test passed.") - return TestResults(totalf, totalt) + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// # Backward compatibility cruft to maintain doctest.master. #///////////////////////////////////////////////////////////////// def merge(self, other): - d = self._name2ft - for name, (f, t) in other._name2ft.items(): + d = self._stats + for name, (failures, tries, skips) in other._stats.items(): if name in d: - # Don't print here by default, since doing - # so breaks some of the buildbots - #print("*** DocTestRunner.merge: '" + name + "' in both" \ - # " testers; summing outcomes.") - f2, t2 = d[name] - f = f + f2 - t = t + t2 - d[name] = f, t + failures2, tries2, skips2 = d[name] + failures = failures + failures2 + tries = tries + tries2 + skips = skips + skips2 + d[name] = (failures, tries, skips) + class OutputChecker: """ @@ -1984,7 +2023,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, @@ -2107,7 +2147,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0): diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 885097c7dda067..06d6b4a3afcd07 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -37,6 +37,8 @@ headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') EMPTYSTRING = '' NL = '\n' +boundaryendRE = re.compile( + r'(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') NeedMoreData = object() @@ -187,7 +189,7 @@ def close(self): assert not self._msgstack # Look for final set of defects if root.get_content_maintype() == 'multipart' \ - and not root.is_multipart(): + and not root.is_multipart() and not self._headersonly: defect = errors.MultipartInvariantViolationDefect() self.policy.handle_defect(root, defect) return root @@ -327,9 +329,10 @@ def _parsegen(self): # this onto the input stream until we've scanned past the # preamble. separator = '--' + boundary - boundaryre = re.compile( - '(?P' + re.escape(separator) + - r')(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') + def boundarymatch(line): + if not line.startswith(separator): + return None + return boundaryendRE.match(line, len(separator)) capturing_preamble = True preamble = [] linesep = False @@ -341,7 +344,7 @@ def _parsegen(self): continue if line == '': break - mo = boundaryre.match(line) + mo = boundarymatch(line) if mo: # If we're looking at the end boundary, we're done with # this multipart. If there was a newline at the end of @@ -373,13 +376,13 @@ def _parsegen(self): if line is NeedMoreData: yield NeedMoreData continue - mo = boundaryre.match(line) + mo = boundarymatch(line) if not mo: self._input.unreadline(line) break # Recurse to parse this subpart; the input stream points # at the subpart's first line. - self._input.push_eof_matcher(boundaryre.match) + self._input.push_eof_matcher(boundarymatch) for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData diff --git a/Lib/email/mime/audio.py b/Lib/email/mime/audio.py index 065819b2a2101d..aa0c4905cbb2b4 100644 --- a/Lib/email/mime/audio.py +++ b/Lib/email/mime/audio.py @@ -6,7 +6,6 @@ __all__ = ['MIMEAudio'] -from io import BytesIO from email import encoders from email.mime.nonmultipart import MIMENonMultipart @@ -59,10 +58,8 @@ def _what(data): # sndhdr.what() had a pretty cruddy interface, unfortunately. This is why # we re-do it here. It would be easier to reverse engineer the Unix 'file' # command and use the standard 'magic' file, as shipped with a modern Unix. - hdr = data[:512] - fakefile = BytesIO(hdr) for testfn in _rules: - if res := testfn(hdr, fakefile): + if res := testfn(data): return res else: return None @@ -74,7 +71,7 @@ def rule(rulefunc): @rule -def _aiff(h, f): +def _aiff(h): if not h.startswith(b'FORM'): return None if h[8:12] in {b'AIFC', b'AIFF'}: @@ -84,7 +81,7 @@ def _aiff(h, f): @rule -def _au(h, f): +def _au(h): if h.startswith(b'.snd'): return 'basic' else: @@ -92,7 +89,7 @@ def _au(h, f): @rule -def _wav(h, f): +def _wav(h): # 'RIFF' 'WAVE' 'fmt ' if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ': return None diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 81da5394ea1695..a49a8fa986ce0c 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -25,8 +25,6 @@ import os import re import time -import random -import socket import datetime import urllib.parse @@ -36,9 +34,6 @@ from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz -# Intrapackage imports -from email.charset import Charset - COMMASPACE = ', ' EMPTYSTRING = '' UEMPTYSTRING = '' @@ -94,6 +89,8 @@ def formataddr(pair, charset='utf-8'): name.encode('ascii') except UnicodeEncodeError: if isinstance(charset, str): + # lazy import to improve module import time + from email.charset import Charset charset = Charset(charset) encoded_name = charset.header_encode(name) return "%s <%s>" % (encoded_name, address) @@ -181,6 +178,11 @@ def make_msgid(idstring=None, domain=None): portion of the message id after the '@'. It defaults to the locally defined hostname. """ + # Lazy imports to speedup module import time + # (no other functions in email.utils need these modules) + import random + import socket + timeval = int(time.time()*100) pid = os.getpid() randint = random.getrandbits(64) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 5f4f1d75b43e64..21e4ad99a39628 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] _PACKAGE_NAMES = ('pip',) -_PIP_VERSION = "23.1.2" +_PIP_VERSION = "23.3.1" _PROJECTS = [ ("pip", _PIP_VERSION, "py3"), ] diff --git a/Lib/ensurepip/_bundled/pip-23.1.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.1.2-py3-none-any.whl deleted file mode 100644 index 6a2515615ccda3..00000000000000 Binary files a/Lib/ensurepip/_bundled/pip-23.1.2-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-23.3.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.3.1-py3-none-any.whl new file mode 100644 index 00000000000000..a4faf716b663e6 Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-23.3.1-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 6e497f7ef6a7de..a8a50a58380375 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,17 +1,16 @@ import sys import builtins as bltns from types import MappingProxyType, DynamicClassAttribute -from operator import or_ as _or_ -from functools import reduce __all__ = [ - 'EnumType', 'EnumMeta', + 'EnumType', 'EnumMeta', 'EnumDict', 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'ReprEnum', 'auto', 'unique', 'property', 'verify', 'member', 'nonmember', 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', 'global_flag_repr', 'global_enum_repr', 'global_str', 'global_enum', 'EnumCheck', 'CONTINUOUS', 'NAMED_FLAGS', 'UNIQUE', + 'pickle_by_global_name', 'pickle_by_enum_name', ] @@ -62,8 +61,8 @@ def _is_sunder(name): return ( len(name) > 2 and name[0] == name[-1] == '_' and - name[1:2] != '_' and - name[-2:-1] != '_' + name[1] != '_' and + name[-2] != '_' ) def _is_internal_class(cls_name, obj): @@ -82,7 +81,6 @@ def _is_private(cls_name, name): if ( len(name) > pat_len and name.startswith(pattern) - and name[pat_len:pat_len+1] != ['_'] and (name[-1] != '_' or name[-2] != '_') ): return True @@ -157,7 +155,6 @@ def _dedent(text): Like textwrap.dedent. Rewritten because we cannot import textwrap. """ lines = text.split('\n') - blanks = 0 for i, ch in enumerate(lines[0]): if ch != ' ': break @@ -316,45 +313,8 @@ def __set_name__(self, enum_class, member_name): ): # no other instances found, record this member in _member_names_ enum_class._member_names_.append(member_name) - # if necessary, get redirect in place and then add it to _member_map_ - found_descriptor = None - descriptor_type = None - class_type = None - for base in enum_class.__mro__[1:]: - attr = base.__dict__.get(member_name) - if attr is not None: - if isinstance(attr, (property, DynamicClassAttribute)): - found_descriptor = attr - class_type = base - descriptor_type = 'enum' - break - elif _is_descriptor(attr): - found_descriptor = attr - descriptor_type = descriptor_type or 'desc' - class_type = class_type or base - continue - else: - descriptor_type = 'attr' - class_type = base - if found_descriptor: - redirect = property() - redirect.member = enum_member - redirect.__set_name__(enum_class, member_name) - if descriptor_type in ('enum','desc'): - # earlier descriptor found; copy fget, fset, fdel to this one. - redirect.fget = getattr(found_descriptor, 'fget', None) - redirect._get = getattr(found_descriptor, '__get__', None) - redirect.fset = getattr(found_descriptor, 'fset', None) - redirect._set = getattr(found_descriptor, '__set__', None) - redirect.fdel = getattr(found_descriptor, 'fdel', None) - redirect._del = getattr(found_descriptor, '__delete__', None) - redirect._attr_type = descriptor_type - redirect._cls_type = class_type - setattr(enum_class, member_name, redirect) - else: - setattr(enum_class, member_name, enum_member) - # now add to _member_map_ (even aliases) - enum_class._member_map_[member_name] = enum_member + + enum_class._add_member_(member_name, enum_member) try: # This may fail if value is not hashable. We can't add the value # to the map, and by-value lookups for this value will be @@ -363,9 +323,10 @@ def __set_name__(self, enum_class, member_name): except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(member_name, []).append(value) -class _EnumDict(dict): +class EnumDict(dict): """ Track enum member order and ensure member names are not reused. @@ -374,7 +335,7 @@ class _EnumDict(dict): """ def __init__(self): super().__init__() - self._member_names = {} # use a dict to keep insertion order + self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False @@ -388,22 +349,15 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_internal_class(self._cls_name, value): - import warnings - warnings.warn( - "In 3.13 classes created inside an enum will not become a member. " - "Use the `member` decorator to keep the current behavior.", - DeprecationWarning, - stacklevel=2, - ) if _is_private(self._cls_name, key): - # also do nothing, name will be a normal attribute + # do nothing, name will be a normal attribute pass elif _is_sunder(key): if key not in ( '_order_', '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', + '_add_alias_', '_add_value_alias_', ): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' @@ -440,10 +394,9 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - # TODO: uncomment next three lines in 3.13 - # elif _is_internal_class(self._cls_name, value): - # # do nothing, name will be a normal attribute - # pass + elif _is_internal_class(self._cls_name, value): + # do nothing, name will be a normal attribute + pass else: if key in self: # enum overwriting a descriptor? @@ -480,6 +433,10 @@ def __setitem__(self, key, value): self._last_values.append(value) super().__setitem__(key, value) + @property + def member_names(self): + return list(self._member_names) + def update(self, members, **more_members): try: for name in members.keys(): @@ -490,6 +447,8 @@ def update(self, members, **more_members): for name, value in more_members.items(): self[name] = value +_EnumDict = EnumDict # keep private name for backwards compatibility + class EnumType(type): """ @@ -501,7 +460,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = _EnumDict() + enum_dict = EnumDict() enum_dict._cls_name = cls # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) @@ -564,6 +523,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} classdict['_unhashable_values_'] = [] + classdict['_unhashable_values_map_'] = {} classdict['_member_type_'] = member_type # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) @@ -580,12 +540,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k try: exc = None enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) - except RuntimeError as e: - # any exceptions raised by member.__new__ will get converted to a - # RuntimeError, so get that original exception back and raise it instead - exc = e.__cause__ or e + except Exception as e: + # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError + # recreate the exception to discard + exc = type(e)(str(e)) + exc.__cause__ = e.__cause__ + exc.__context__ = e.__context__ + tb = e.__traceback__ if exc is not None: - raise exc + raise exc.with_traceback(tb) # # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__) @@ -738,6 +702,11 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type + if names is None and type is None: + # no body? no data-type? possibly wrong usage + raise TypeError( + f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" + ) return cls._create_( class_name=value, names=names, @@ -757,7 +726,10 @@ def __contains__(cls, value): """ if isinstance(value, cls): return True - return value in cls._value2member_map_ or value in cls._unhashable_values_ + try: + return value in cls._value2member_map_ + except TypeError: + return value in cls._unhashable_values_ def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -864,6 +836,8 @@ def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, s value = first_enum._generate_next_value_(name, start, count, last_values[:]) last_values.append(value) names.append((name, value)) + if names is None: + names = () # Here, names is either an iterable of (name, value) or a mapping. for item in names: @@ -922,7 +896,6 @@ def _convert_(cls, name, module, filter, source=None, *, boundary=None, as_globa body['__module__'] = module tmp_cls = type(name, (object, ), body) cls = _simple_enum(etype=cls, boundary=boundary or KEEP)(tmp_cls) - cls.__reduce_ex__ = _reduce_ex_by_global_name if as_global: global_enum(cls) else: @@ -1052,7 +1025,57 @@ def _find_new_(mcls, classdict, member_type, first_enum): else: use_args = True return __new__, save_new, use_args -EnumMeta = EnumType + + def _add_member_(cls, name, member): + # _value_ structures are not updated + if name in cls._member_map_: + if cls._member_map_[name] is not member: + raise NameError('%r is already bound: %r' % (name, cls._member_map_[name])) + return + # + # if necessary, get redirect in place and then add it to _member_map_ + found_descriptor = None + descriptor_type = None + class_type = None + for base in cls.__mro__[1:]: + attr = base.__dict__.get(name) + if attr is not None: + if isinstance(attr, (property, DynamicClassAttribute)): + found_descriptor = attr + class_type = base + descriptor_type = 'enum' + break + elif _is_descriptor(attr): + found_descriptor = attr + descriptor_type = descriptor_type or 'desc' + class_type = class_type or base + continue + else: + descriptor_type = 'attr' + class_type = base + if found_descriptor: + redirect = property() + redirect.member = member + redirect.__set_name__(cls, name) + if descriptor_type in ('enum', 'desc'): + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = getattr(found_descriptor, 'fget', None) + redirect._get = getattr(found_descriptor, '__get__', None) + redirect.fset = getattr(found_descriptor, 'fset', None) + redirect._set = getattr(found_descriptor, '__set__', None) + redirect.fdel = getattr(found_descriptor, 'fdel', None) + redirect._del = getattr(found_descriptor, '__delete__', None) + redirect._attr_type = descriptor_type + redirect._cls_type = class_type + setattr(cls, name, redirect) + else: + setattr(cls, name, member) + # now add to _member_map_ (even aliases) + cls._member_map_[name] = member + # + cls._member_map_[name] = member + +EnumMeta = EnumType # keep EnumMeta name for backwards compatibility class Enum(metaclass=EnumType): @@ -1118,9 +1141,14 @@ def __new__(cls, value): pass except TypeError: # not there, now do long search -- O(n) behavior - for member in cls._member_map_.values(): - if member._value_ == value: - return member + for name, values in cls._unhashable_values_map_.items(): + if value in values: + return cls[name] + # still not found -- verify that members exist, in-case somebody got here mistakenly + # (such as via super when trying to override __new__) + if not cls._member_map_: + raise TypeError("%r has no members defined" % cls) + # # still not found -- try _missing_ hook try: exc = None @@ -1156,6 +1184,33 @@ def __new__(cls, value): def __init__(self, *args, **kwds): pass + def _add_alias_(self, name): + self.__class__._add_member_(name, self) + + def _add_value_alias_(self, value): + cls = self.__class__ + try: + if value in cls._value2member_map_: + if cls._value2member_map_[value] is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + except TypeError: + # unhashable value, do long search + for m in cls._member_map_.values(): + if m._value_ == value: + if m is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + cls._value2member_map_.setdefault(value, self) + except TypeError: + # keep track of the value in a list so containment checks are quick + cls._unhashable_values_.append(value) + cls._unhashable_values_map_.setdefault(self.name, []).append(value) + @staticmethod def _generate_next_value_(name, start, count, last_values): """ @@ -1169,28 +1224,13 @@ def _generate_next_value_(name, start, count, last_values): if not last_values: return start try: - last = last_values[-1] - last_values.sort() - if last == last_values[-1]: - # no difference between old and new methods - return last + 1 - else: - # trigger old method (with warning) - raise TypeError + last_value = sorted(last_values).pop() except TypeError: - import warnings - warnings.warn( - "In 3.13 the default `auto()`/`_generate_next_value_` will require all values to be sortable and support adding +1\n" - "and the value returned will be the largest value in the enum incremented by 1", - DeprecationWarning, - stacklevel=3, - ) - for v in last_values: - try: - return v + 1 - except TypeError: - pass - return start + raise TypeError('unable to sort non-numeric values') from None + try: + return last_value + 1 + except TypeError: + raise TypeError('unable to increment %r' % (last_value, )) from None @classmethod def _missing_(cls, value): @@ -1205,14 +1245,13 @@ def __str__(self): def __dir__(self): """ - Returns all members and all public methods + Returns public methods and other interesting attributes. """ - if self.__class__._member_type_ is object: - interesting = set(['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'value']) - else: + interesting = set() + if self.__class__._member_type_ is not object: interesting = set(object.__dir__(self)) for name in getattr(self, '__dict__', []): - if name[0] != '_': + if name[0] != '_' and name not in self._member_map_: interesting.add(name) for cls in self.__class__.mro(): for name, obj in cls.__dict__.items(): @@ -1225,7 +1264,7 @@ def __dir__(self): else: # in case it was added by `dir(self)` interesting.discard(name) - else: + elif name not in self._member_map_: interesting.add(name) names = sorted( set(['__class__', '__doc__', '__eq__', '__hash__', '__module__']) @@ -1240,7 +1279,13 @@ def __hash__(self): return hash(self._name_) def __reduce_ex__(self, proto): - return getattr, (self.__class__, self._name_) + return self.__class__, (self._value_, ) + + def __deepcopy__(self,memo): + return self + + def __copy__(self): + return self # enum.property is used to provide access to the `name` and # `value` attributes of enum members while keeping some measure of @@ -1307,8 +1352,14 @@ def _generate_next_value_(name, start, count, last_values): return name.lower() -def _reduce_ex_by_global_name(self, proto): +def pickle_by_global_name(self, proto): + # should not be used with Flag-type enums return self.name +_reduce_ex_by_global_name = pickle_by_global_name + +def pickle_by_enum_name(self, proto): + # should not be used with Flag-type enums + return getattr, (self.__class__, self._name_) class FlagBoundary(StrEnum): """ @@ -1330,23 +1381,6 @@ class Flag(Enum, boundary=STRICT): Support for flags """ - def __reduce_ex__(self, proto): - cls = self.__class__ - unknown = self._value_ & ~cls._flag_mask_ - member_value = self._value_ & cls._flag_mask_ - if unknown and member_value: - return _or_, (cls(member_value), unknown) - for val in _iter_bits_lsb(member_value): - rest = member_value & ~val - if rest: - return _or_, (cls(rest), cls._value2member_map_.get(val)) - else: - break - if self._name_ is None: - return cls, (self._value_,) - else: - return getattr, (cls, self._name_) - _numeric_repr_ = repr @staticmethod @@ -1474,12 +1508,11 @@ def _missing_(cls, value): else: pseudo_member._name_ = None # use setdefault in case another thread already created a composite - # with this value, but only if all members are known - # note: zero is a special case -- add it - if not unknown: - pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) - if neg_value is not None: - cls._value2member_map_[neg_value] = pseudo_member + # with this value + # note: zero is a special case -- always add it + pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) + if neg_value is not None: + cls._value2member_map_[neg_value] = pseudo_member return pseudo_member def __contains__(self, other): @@ -1551,14 +1584,10 @@ def __xor__(self, other): def __invert__(self): if self._inverted_ is None: - if self._boundary_ is KEEP: - # use all bits + if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) else: - # calculate flags not in this member - self._inverted_ = self.__class__(self._flag_mask_ ^ self._value_) - if isinstance(self._inverted_, self.__class__): - self._inverted_._inverted_ = self + self._inverted_ = self.__class__(self._singles_mask_ & ~self._value_) return self._inverted_ __rand__ = __and__ @@ -1663,7 +1692,7 @@ def _simple_enum(etype=Enum, *, boundary=None, use_args=None): Class decorator that converts a normal class into an :class:`Enum`. No safety checks are done, and some advanced behavior (such as :func:`__init_subclass__`) is not available. Enum creation can be faster - using :func:`simple_enum`. + using :func:`_simple_enum`. >>> from enum import Enum, _simple_enum >>> @_simple_enum(Enum) @@ -1694,7 +1723,8 @@ def convert_class(cls): body['_member_names_'] = member_names = [] body['_member_map_'] = member_map = {} body['_value2member_map_'] = value2member_map = {} - body['_unhashable_values_'] = [] + body['_unhashable_values_'] = unhashable_values = [] + body['_unhashable_values_map_'] = {} body['_member_type_'] = member_type = etype._member_type_ body['_value_repr_'] = etype._value_repr_ if issubclass(etype, Flag): @@ -1741,14 +1771,9 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map: + if value in value2member_map or value in unhashable_values: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + enum_class(value)._add_alias_(name) else: # create the member if use_args: @@ -1763,12 +1788,12 @@ def convert_class(cls): member._name_ = name member.__objclass__ = enum_class member.__init__(value) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member member._sort_order_ = len(member_names) + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) value2member_map[value] = member if _is_single_bit(value): # not a multi-bit alias, record in _member_names_ and _flag_mask_ @@ -1791,14 +1816,13 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value - if value in value2member_map: + try: + contained = value in value2member_map + except TypeError: + contained = value in unhashable_values + if contained: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + enum_class(value)._add_alias_(name) else: # create the member if use_args: @@ -1814,14 +1838,22 @@ def convert_class(cls): member.__objclass__ = enum_class member.__init__(value) member._sort_order_ = len(member_names) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member - value2member_map[value] = member + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) member_names.append(name) gnv_last_values.append(value) + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, member) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(name, []).append(value) if '__new__' in body: enum_class.__new_member__ = enum_class.__new__ enum_class.__new__ = Enum.__new__ @@ -1906,7 +1938,8 @@ def __call__(self, enumeration): missed = [v for v in values if v not in member_values] if missed: missing_names.append(name) - missing_value |= reduce(_or_, missed) + for val in missed: + missing_value |= val if missing_names: if len(missing_names) == 1: alias = 'alias %s is missing' % missing_names[0] @@ -2073,7 +2106,6 @@ def _old_convert_(etype, name, module, filter, source=None, *, boundary=None): # unless some values aren't comparable, in which case sort by name members.sort(key=lambda t: t[0]) cls = etype(name, members, module=module, boundary=boundary or KEEP) - cls.__reduce_ex__ = _reduce_ex_by_global_name return cls _stdlib_enums = IntEnum, StrEnum, IntFlag diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index d5e296f7748c1c..73acb1fe8d4106 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -78,6 +78,11 @@ def translate(pat): """ STAR = object() + parts = _translate(pat, STAR, '.') + return _join_translated_parts(parts, STAR) + + +def _translate(pat, STAR, QUESTION_MARK): res = [] add = res.append i, n = 0, len(pat) @@ -89,7 +94,7 @@ def translate(pat): if (not res) or res[-1] is not STAR: add(STAR) elif c == '?': - add('.') + add(QUESTION_MARK) elif c == '[': j = i if j < n and pat[j] == '!': @@ -146,9 +151,11 @@ def translate(pat): else: add(re.escape(c)) assert i == n + return res + +def _join_translated_parts(inp, STAR): # Deal with STARs. - inp = res res = [] add = res.append i, n = 0, len(inp) diff --git a/Lib/functools.py b/Lib/functools.py index aaf4291150fbbf..55990e742bf23f 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -19,8 +19,9 @@ # import types, weakref # Deferred to single_dispatch() from reprlib import recursive_repr from _thread import RLock -from types import GenericAlias +# Avoid importing types, so we can speedup import time +GenericAlias = type(list[int]) ################################################################################ ### update_wrapper() and wraps() decorator @@ -30,7 +31,7 @@ # wrapper functions that can handle naive introspection WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', - '__annotations__') + '__annotations__', '__type_params__') WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, @@ -236,7 +237,7 @@ def __ge__(self, other): def reduce(function, sequence, initial=_initial_missing): """ - reduce(function, iterable[, initial]) -> value + reduce(function, iterable[, initial], /) -> value Apply a function of two arguments cumulatively to the items of a sequence or iterable, from left to right, so as to reduce the iterable to a single @@ -483,8 +484,9 @@ def lru_cache(maxsize=128, typed=False): can grow without bound. If *typed* is True, arguments of different types will be cached separately. - For example, f(3.0) and f(3) will be treated as distinct calls with - distinct results. + For example, f(decimal.Decimal("3.0")) and f(3.0) will be treated as + distinct calls with distinct results. Some types such as str and int may + be cached separately even when typed is false. Arguments to the cached function must be hashable. @@ -933,6 +935,9 @@ def __init__(self, func): self.dispatcher = singledispatch(func) self.func = func + import weakref # see comment in singledispatch function + self._method_cache = weakref.WeakKeyDictionary() + def register(self, cls, method=None): """generic_method.register(cls, func) -> func @@ -941,13 +946,27 @@ def register(self, cls, method=None): return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): + if self._method_cache is not None: + try: + _method = self._method_cache[obj] + except TypeError: + self._method_cache = None + except KeyError: + pass + else: + return _method + + dispatch = self.dispatcher.dispatch def _method(*args, **kwargs): - method = self.dispatcher.dispatch(args[0].__class__) - return method.__get__(obj, cls)(*args, **kwargs) + return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.register = self.register update_wrapper(_method, self.func) + + if self._method_cache is not None: + self._method_cache[obj] = _method + return _method @property @@ -956,15 +975,17 @@ def __isabstractmethod__(self): ################################################################################ -### cached_property() - computed once per instance, cached as attribute +### cached_property() - property result cached as instance attribute ################################################################################ +_NOT_FOUND = object() class cached_property: def __init__(self, func): self.func = func self.attrname = None self.__doc__ = func.__doc__ + self.__module__ = func.__module__ def __set_name__(self, owner, name): if self.attrname is None: @@ -989,15 +1010,17 @@ def __get__(self, instance, owner=None): f"instance to cache {self.attrname!r} property." ) raise TypeError(msg) from None - val = self.func(instance) - try: - cache[self.attrname] = val - except TypeError: - msg = ( - f"The '__dict__' attribute on {type(instance).__name__!r} instance " - f"does not support item assignment for caching {self.attrname!r} property." - ) - raise TypeError(msg) from None + val = cache.get(self.attrname, _NOT_FOUND) + if val is _NOT_FOUND: + val = self.func(instance) + try: + cache[self.attrname] = val + except TypeError: + msg = ( + f"The '__dict__' attribute on {type(instance).__name__!r} instance " + f"does not support item assignment for caching {self.attrname!r} property." + ) + raise TypeError(msg) from None return val __class_getitem__ = classmethod(GenericAlias) diff --git a/Lib/getpass.py b/Lib/getpass.py index 6970d8adfbab36..bd0097ced94c5e 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -18,7 +18,6 @@ import io import os import sys -import warnings __all__ = ["getpass","getuser","GetPassWarning"] @@ -118,6 +117,7 @@ def win_getpass(prompt='Password: ', stream=None): def fallback_getpass(prompt='Password: ', stream=None): + import warnings warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: @@ -156,7 +156,11 @@ def getuser(): First try various environment variables, then the password database. This works on Windows as long as USERNAME is set. + Any failure to find a username raises OSError. + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` + were raised. """ for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): @@ -164,9 +168,12 @@ def getuser(): if user: return user - # If this fails, the exception will "explain" why - import pwd - return pwd.getpwuid(os.getuid())[0] + try: + import pwd + return pwd.getpwuid(os.getuid())[0] + except (ImportError, KeyError) as e: + raise OSError('No username set in the environment') from e + # Bind the name getpass to the appropriate function try: diff --git a/Lib/gettext.py b/Lib/gettext.py index 6c5ec4e517f637..62cff81b7b3d49 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -46,6 +46,7 @@ # find this format documented anywhere. +import operator import os import re import sys @@ -166,14 +167,28 @@ def _parse(tokens, priority=-1): def _as_int(n): try: - i = round(n) + round(n) except TypeError: raise TypeError('Plural value must be an integer, got %s' % (n.__class__.__name__,)) from None + return _as_int2(n) + +def _as_int2(n): + try: + return operator.index(n) + except TypeError: + pass + import warnings + frame = sys._getframe(1) + stacklevel = 2 + while frame.f_back is not None and frame.f_globals.get('__name__') == __name__: + stacklevel += 1 + frame = frame.f_back warnings.warn('Plural value must be an integer, got %s' % (n.__class__.__name__,), - DeprecationWarning, 4) + DeprecationWarning, + stacklevel) return n @@ -200,7 +215,7 @@ def c2py(plural): elif c == ')': depth -= 1 - ns = {'_as_int': _as_int} + ns = {'_as_int': _as_int, '__name__': __name__} exec('''if True: def func(n): if not isinstance(n, int): @@ -280,6 +295,7 @@ def gettext(self, message): def ngettext(self, msgid1, msgid2, n): if self._fallback: return self._fallback.ngettext(msgid1, msgid2, n) + n = _as_int2(n) if n == 1: return msgid1 else: @@ -293,6 +309,7 @@ def pgettext(self, context, message): def npgettext(self, context, msgid1, msgid2, n): if self._fallback: return self._fallback.npgettext(context, msgid1, msgid2, n) + n = _as_int2(n) if n == 1: return msgid1 else: @@ -422,10 +439,12 @@ def gettext(self, message): missing = object() tmsg = self._catalog.get(message, missing) if tmsg is missing: - if self._fallback: - return self._fallback.gettext(message) - return message - return tmsg + tmsg = self._catalog.get((message, self.plural(1)), missing) + if tmsg is not missing: + return tmsg + if self._fallback: + return self._fallback.gettext(message) + return message def ngettext(self, msgid1, msgid2, n): try: @@ -444,10 +463,12 @@ def pgettext(self, context, message): missing = object() tmsg = self._catalog.get(ctxt_msg_id, missing) if tmsg is missing: - if self._fallback: - return self._fallback.pgettext(context, message) - return message - return tmsg + tmsg = self._catalog.get((ctxt_msg_id, self.plural(1)), missing) + if tmsg is not missing: + return tmsg + if self._fallback: + return self._fallback.pgettext(context, message) + return message def npgettext(self, context, msgid1, msgid2, n): ctxt_msg_id = self.CONTEXT % (context, msgid1) @@ -575,6 +596,7 @@ def dngettext(domain, msgid1, msgid2, n): try: t = translation(domain, _localedirs.get(domain, None)) except OSError: + n = _as_int2(n) if n == 1: return msgid1 else: @@ -594,6 +616,7 @@ def dnpgettext(domain, context, msgid1, msgid2, n): try: t = translation(domain, _localedirs.get(domain, None)) except OSError: + n = _as_int2(n) if n == 1: return msgid1 else: diff --git a/Lib/glob.py b/Lib/glob.py index a7256422d520fb..4a335a10766cf4 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -249,3 +249,63 @@ def escape(pathname): _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) + + +def translate(pat, *, recursive=False, include_hidden=False, seps=None): + """Translate a pathname with shell wildcards to a regular expression. + + If `recursive` is true, the pattern segment '**' will match any number of + path segments; if '**' appears outside its own segment, ValueError will be + raised. + + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). + + If a sequence of separator characters is given to `seps`, they will be + used to split the pattern into segments and match path separators. If not + given, os.path.sep and os.path.altsep (where available) are used. + """ + if not seps: + if os.path.altsep: + seps = (os.path.sep, os.path.altsep) + else: + seps = os.path.sep + escaped_seps = ''.join(map(re.escape, seps)) + any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps + not_sep = f'[^{escaped_seps}]' + if include_hidden: + one_last_segment = f'{not_sep}+' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:.+{any_sep})?' + any_last_segments = '.*' + else: + one_last_segment = f'[^{escaped_seps}.]{not_sep}*' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:{one_segment})*' + any_last_segments = f'{any_segments}(?:{one_last_segment})?' + + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == '*': + results.append(one_segment if idx < last_part_idx else one_last_segment) + continue + if recursive: + if part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + continue + elif '**' in part: + raise ValueError("Invalid pattern: '**' can only be an entire path component") + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + if idx < last_part_idx: + results.append(any_sep) + res = ''.join(results) + return fr'(?s:{res})\Z' diff --git a/Lib/gzip.py b/Lib/gzip.py index 8796c8d9fd9a2d..177f9080dc5af8 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -370,8 +370,9 @@ def close(self): def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: - # Ensure the compressor's buffer is flushed self._buffer.flush() + # Ensure the compressor's buffer is flushed + self.fileobj.write(self.compress.flush(zlib_mode)) self.fileobj.flush() def fileno(self): @@ -400,6 +401,9 @@ def seekable(self): def seek(self, offset, whence=io.SEEK_SET): if self.mode == WRITE: + self._check_not_closed() + # Flush buffer to ensure validity of self.offset + self._buffer.flush() if whence != io.SEEK_SET: if whence == io.SEEK_CUR: offset = self.offset + offset diff --git a/Lib/http/client.py b/Lib/http/client.py index 50f2b4680769c8..7bb5d824bb6da4 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -172,6 +172,13 @@ def _encode(data, name='data'): "if you want to send it encoded in UTF-8." % (name.title(), data[err.start:err.end], name)) from None +def _strip_ipv6_iface(enc_name: bytes) -> bytes: + """Remove interface scope from IPv6 address.""" + enc_name, percent, _ = enc_name.partition(b"%") + if percent: + assert enc_name.startswith(b'['), enc_name + enc_name += b']' + return enc_name class HTTPMessage(email.message.Message): # XXX The only usage of this method is in @@ -221,8 +228,9 @@ def _read_headers(fp): break return headers -def parse_headers(fp, _class=HTTPMessage): - """Parses only RFC2822 headers from a file pointer. +def _parse_header_lines(header_lines, _class=HTTPMessage): + """ + Parses only RFC2822 headers from header lines. email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many bytes @@ -231,10 +239,15 @@ def parse_headers(fp, _class=HTTPMessage): to parse. """ - headers = _read_headers(fp) - hstring = b''.join(headers).decode('iso-8859-1') + hstring = b''.join(header_lines).decode('iso-8859-1') return email.parser.Parser(_class=_class).parsestr(hstring) +def parse_headers(fp, _class=HTTPMessage): + """Parses only RFC2822 headers from a file pointer.""" + + headers = _read_headers(fp) + return _parse_header_lines(headers, _class) + class HTTPResponse(io.BufferedIOBase): @@ -858,7 +871,7 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self._tunnel_host = None self._tunnel_port = None self._tunnel_headers = {} - self._proxy_response_headers = None + self._raw_proxy_headers = None (self.host, self.port) = self._get_hostport(host, port) @@ -945,11 +958,11 @@ def _tunnel(self): try: (version, code, message) = response._read_status() - self._proxy_response_headers = parse_headers(response.fp) + self._raw_proxy_headers = _read_headers(response.fp) if self.debuglevel > 0: - for hdr, val in self._proxy_response_headers.items(): - print("header:", hdr + ":", val) + for header in self._raw_proxy_headers: + print('header:', header.decode()) if code != http.HTTPStatus.OK: self.close() @@ -958,6 +971,20 @@ def _tunnel(self): finally: response.close() + def get_proxy_response_headers(self): + """ + Returns a dictionary with the headers of the response + received from the proxy server to the CONNECT request + sent to set the tunnel. + + If the CONNECT request was not sent, the method returns None. + """ + return ( + _parse_header_lines(self._raw_proxy_headers) + if self._raw_proxy_headers is not None + else None + ) + def connect(self): """Connect to the host and port specified in __init__.""" sys.audit("http.client.connect", self, self.host, self.port) @@ -1003,7 +1030,7 @@ def send(self, data): print("send:", repr(data)) if hasattr(data, "read") : if self.debuglevel > 0: - print("sendIng a read()able") + print("sending a readable") encode = self._is_textIO(data) if encode and self.debuglevel > 0: print("encoding file using iso-8859-1") @@ -1033,7 +1060,7 @@ def _output(self, s): def _read_readable(self, readable): if self.debuglevel > 0: - print("sendIng a read()able") + print("reading a readable") encode = self._is_textIO(readable) if encode and self.debuglevel > 0: print("encoding file using iso-8859-1") @@ -1174,7 +1201,7 @@ def putrequest(self, method, url, skip_host=False, netloc_enc = netloc.encode("ascii") except UnicodeEncodeError: netloc_enc = netloc.encode("idna") - self.putheader('Host', netloc_enc) + self.putheader('Host', _strip_ipv6_iface(netloc_enc)) else: if self._tunnel_host: host = self._tunnel_host @@ -1191,8 +1218,9 @@ def putrequest(self, method, url, skip_host=False, # As per RFC 273, IPv6 address should be wrapped with [] # when used as Host header - if host.find(':') >= 0: + if ":" in host: host_enc = b'[' + host_enc + b']' + host_enc = _strip_ipv6_iface(host_enc) if port == self.default_port: self.putheader('Host', host_enc) diff --git a/Lib/http/server.py b/Lib/http/server.py index ca6240d9a921e6..ee7a9b6aa55b88 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -2,18 +2,18 @@ Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST, -and CGIHTTPRequestHandler for CGI scripts. +and (deprecated) CGIHTTPRequestHandler for CGI scripts. -It does, however, optionally implement HTTP/1.1 persistent connections, -as of version 0.3. +It does, however, optionally implement HTTP/1.1 persistent connections. Notes on CGIHTTPRequestHandler ------------------------------ -This class implements GET and POST requests to cgi-bin scripts. +This class is deprecated. It implements GET and POST requests to cgi-bin scripts. -If the os.fork() function is not present (e.g. on Windows), -subprocess.Popen() is used as a fallback, with slightly altered semantics. +If the os.fork() function is not present (Windows), subprocess.Popen() is used, +with slightly altered but never documented semantics. Use from a threaded +process is likely to trigger a warning at os.fork() time. In all cases, the implementation is intentionally naive -- all requests are executed synchronously. @@ -986,6 +986,12 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): """ + def __init__(self, *args, **kwargs): + import warnings + warnings._deprecated("http.server.CGIHTTPRequestHandler", + remove=(3, 15)) + super().__init__(*args, **kwargs) + # Determine platform specifics have_fork = hasattr(os, 'fork') diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index e97e3ca01ef30c..f258797c6e0fb3 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -4,6 +4,14 @@ Released on 2023-10-02 ========================= +gh-104719: Remove IDLE's modification of tokenize.tabsize and test +other uses of tokenize data and methods. + +gh-104499: Fix completions for Tk Aqua 8.7 (currently blank). + +gh-104486: Make About print both tcl and tk versions if different, +as is expected someday. + gh-88496 Fix IDLE test hang on macOS. gh-103314 Support sys.last_exc after exceptions in Shell. diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py index 0f835a9cc1d010..24320b5a3bffab 100644 --- a/Lib/idlelib/autocomplete_w.py +++ b/Lib/idlelib/autocomplete_w.py @@ -182,16 +182,11 @@ def show_window(self, comp_lists, index, complete, mode, userWantsWin): self.userwantswindow = userWantsWin self.lasttypedstart = self.start - # Put widgets in place self.autocompletewindow = acw = Toplevel(self.widget) - # Put it in a position so that it is not seen. - acw.wm_geometry("+10000+10000") - # Make it float + acw.withdraw() acw.wm_overrideredirect(1) try: - # This command is only needed and available on Tk >= 8.4.0 for OSX - # Without it, call tips intrude on the typing process by grabbing - # the focus. + # Prevent grabbing focus on macOS. acw.tk.call("::tk::unsupported::MacWindowStyle", "style", acw._w, "help", "noActivates") except TclError: @@ -271,6 +266,7 @@ def winconfig_event(self, event): # place acw above current line new_y -= acw_height acw.wm_geometry("+%d+%d" % (new_x, new_y)) + acw.deiconify() acw.update_idletasks() except TclError: pass diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 4fe64dced60aca..8b9060e57072ea 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -250,9 +250,11 @@ def closure(): class Nested_in_closure: pass ModuleBrowser(parent, file, _htest=True) + if __name__ == "__main__": if len(sys.argv) == 1: # If pass file on command line, unittest fails. from unittest import main main('idlelib.idle_test.test_browser', verbosity=2, exit=False) + from idlelib.idle_test.htest import run run(_module_browser) diff --git a/Lib/idlelib/calltip_w.py b/Lib/idlelib/calltip_w.py index 278546064adde2..9386376058c791 100644 --- a/Lib/idlelib/calltip_w.py +++ b/Lib/idlelib/calltip_w.py @@ -193,6 +193,7 @@ def calltip_hide(event): text.focus_set() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_calltip_w', verbosity=2, exit=False) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 2b09d79470b47c..92992fd9cce9cd 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -597,7 +597,9 @@ def GetCoreKeys(self, keySetName=None): problem getting any core binding there will be an 'ultimate last resort fallback' to the CUA-ish bindings defined here. """ + # TODO: = dict(sorted([(v-event, keys), ...]))? keyBindings={ + # vitual-event: list of key events. '<>': ['', ''], '<>': ['', ''], '<>': ['', ''], @@ -880,7 +882,7 @@ def _dump(): # htest # (not really, but ignore in coverage) line, crc = 0, 0 def sprint(obj): - global line, crc + nonlocal line, crc txt = str(obj) line += 1 crc = crc32(txt.encode(encoding='utf-8'), crc) @@ -889,7 +891,7 @@ def sprint(obj): def dumpCfg(cfg): print('\n', cfg, '\n') # Cfg has variable '0xnnnnnnnn' address. - for key in sorted(cfg.keys()): + for key in sorted(cfg): sections = cfg[key].sections() sprint(key) sprint(sections) @@ -904,8 +906,11 @@ def dumpCfg(cfg): dumpCfg(idleConf.userCfg) print('\nlines = ', line, ', crc = ', crc, sep='') + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_config', verbosity=2, exit=False) - # Run revised _dump() as htest? + _dump() + # Run revised _dump() (700+ lines) as htest? More sorting. + # Perhaps as window with tabs for textviews, making it config viewer. diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index bb07231cd590b6..e5f67e8d4069ee 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -351,4 +351,4 @@ def cancel(self, event=None): main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(GetKeysDialog) + run(GetKeysWindow) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index cda7966d558a51..eedf97bf74fe6a 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -211,14 +211,8 @@ def help(self): contents=help_common+help_pages.get(page, '')) def deactivate_current_config(self): - """Remove current key bindings. - Iterate over window instances defined in parent and remove - the keybindings. - """ - # Before a config is saved, some cleanup of current - # config must be done - remove the previous keybindings. - win_instances = self.parent.instance_dict.keys() - for instance in win_instances: + """Remove current key bindings in current windows.""" + for instance in self.parent.instance_dict: instance.RemoveKeybindings() def activate_config_changes(self): @@ -227,8 +221,7 @@ def activate_config_changes(self): Dynamically update the current parent window instances with some of the configuration changes. """ - win_instances = self.parent.instance_dict.keys() - for instance in win_instances: + for instance in self.parent.instance_dict: instance.ResetColorizer() instance.ResetFont() instance.set_notabs_indentwidth() @@ -583,22 +576,23 @@ def create_page_highlight(self): (*)theme_message: Label """ self.theme_elements = { - 'Normal Code or Text': ('normal', '00'), - 'Code Context': ('context', '01'), - 'Python Keywords': ('keyword', '02'), - 'Python Definitions': ('definition', '03'), - 'Python Builtins': ('builtin', '04'), - 'Python Comments': ('comment', '05'), - 'Python Strings': ('string', '06'), - 'Selected Text': ('hilite', '07'), - 'Found Text': ('hit', '08'), - 'Cursor': ('cursor', '09'), - 'Editor Breakpoint': ('break', '10'), - 'Shell Prompt': ('console', '11'), - 'Error Text': ('error', '12'), - 'Shell User Output': ('stdout', '13'), - 'Shell User Exception': ('stderr', '14'), - 'Line Number': ('linenumber', '16'), + # Display-name: internal-config-tag-name. + 'Normal Code or Text': 'normal', + 'Code Context': 'context', + 'Python Keywords': 'keyword', + 'Python Definitions': 'definition', + 'Python Builtins': 'builtin', + 'Python Comments': 'comment', + 'Python Strings': 'string', + 'Selected Text': 'hilite', + 'Found Text': 'hit', + 'Cursor': 'cursor', + 'Editor Breakpoint': 'break', + 'Shell Prompt': 'console', + 'Error Text': 'error', + 'Shell User Output': 'stdout', + 'Shell User Exception': 'stderr', + 'Line Number': 'linenumber', } self.builtin_name = tracers.add( StringVar(self), self.var_changed_builtin_name) @@ -658,7 +652,7 @@ def tem(event, elem=element): # event.widget.winfo_top_level().highlight_target.set(elem) self.highlight_target.set(elem) text.tag_bind( - self.theme_elements[element][0], '', tem) + self.theme_elements[element], '', tem) text['state'] = 'disabled' self.style.configure('frame_color_set.TFrame', borderwidth=1, relief='solid') @@ -765,8 +759,7 @@ def load_theme_cfg(self): self.builtinlist.SetMenu(item_list, item_list[0]) self.set_theme_type() # Load theme element option menu. - theme_names = list(self.theme_elements.keys()) - theme_names.sort(key=lambda x: self.theme_elements[x][1]) + theme_names = list(self.theme_elements) self.targetlist.SetMenu(theme_names, theme_names[0]) self.paint_theme_sample() self.set_highlight_target() @@ -893,7 +886,7 @@ def on_new_color_set(self): new_color = self.color.get() self.style.configure('frame_color_set.TFrame', background=new_color) plane = 'foreground' if self.fg_bg_toggle.get() else 'background' - sample_element = self.theme_elements[self.highlight_target.get()][0] + sample_element = self.theme_elements[self.highlight_target.get()] self.highlight_sample.tag_config(sample_element, **{plane: new_color}) theme = self.custom_name.get() theme_element = sample_element + '-' + plane @@ -1007,7 +1000,7 @@ def set_color_sample(self): frame_color_set """ # Set the color sample area. - tag = self.theme_elements[self.highlight_target.get()][0] + tag = self.theme_elements[self.highlight_target.get()] plane = 'foreground' if self.fg_bg_toggle.get() else 'background' color = self.highlight_sample.tag_cget(tag, plane) self.style.configure('frame_color_set.TFrame', background=color) @@ -1037,7 +1030,7 @@ def paint_theme_sample(self): else: # User theme theme = self.custom_name.get() for element_title in self.theme_elements: - element = self.theme_elements[element_title][0] + element = self.theme_elements[element_title] colors = idleConf.GetHighlight(theme, element) if element == 'cursor': # Cursor sample needs special painting. colors['background'] = idleConf.GetHighlight( @@ -1477,12 +1470,13 @@ def load_keys_list(self, keyset_name): reselect = True list_index = self.bindingslist.index(ANCHOR) keyset = idleConf.GetKeySet(keyset_name) - bind_names = list(keyset.keys()) + # 'set' is dict mapping virtual event to list of key events. + bind_names = list(keyset) bind_names.sort() self.bindingslist.delete(0, END) for bind_name in bind_names: key = ' '.join(keyset[bind_name]) - bind_name = bind_name[2:-2] # Trim off the angle brackets. + bind_name = bind_name[2:-2] # Trim double angle brackets. if keyset_name in changes['keys']: # Handle any unsaved changes to this key set. if bind_name in changes['keys'][keyset_name]: diff --git a/Lib/idlelib/debugger.py b/Lib/idlelib/debugger.py index 452c62b42655b3..f487b4c4b16a60 100644 --- a/Lib/idlelib/debugger.py +++ b/Lib/idlelib/debugger.py @@ -1,3 +1,20 @@ +"""Debug user code with a GUI interface to a subclass of bdb.Bdb. + +The Idb idb and Debugger gui instances each need a reference to each +other or to an rpc proxy for each other. + +If IDLE is started with '-n', so that user code and idb both run in the +IDLE process, Debugger is called without an idb. Debugger.__init__ +calls Idb with its incomplete self. Idb.__init__ stores gui and gui +then stores idb. + +If IDLE is started normally, so that user code executes in a separate +process, debugger_r.start_remote_debugger is called, executing in the +IDLE process. It calls 'start the debugger' in the remote process, +which calls Idb with a gui proxy. Then Debugger is called in the IDLE +for more. +""" + import bdb import os @@ -10,66 +27,95 @@ class Idb(bdb.Bdb): + "Supply user_line and user_exception functions for Bdb." def __init__(self, gui): - self.gui = gui # An instance of Debugger or proxy of remote. - bdb.Bdb.__init__(self) + self.gui = gui # An instance of Debugger or proxy thereof. + super().__init__() def user_line(self, frame): - if self.in_rpc_code(frame): + """Handle a user stopping or breaking at a line. + + Convert frame to a string and send it to gui. + """ + if _in_rpc_code(frame): self.set_step() return - message = self.__frame2message(frame) + message = _frame2message(frame) try: self.gui.interaction(message, frame) except TclError: # When closing debugger window with [x] in 3.x pass - def user_exception(self, frame, info): - if self.in_rpc_code(frame): + def user_exception(self, frame, exc_info): + """Handle an the occurrence of an exception.""" + if _in_rpc_code(frame): self.set_step() return - message = self.__frame2message(frame) - self.gui.interaction(message, frame, info) - - def in_rpc_code(self, frame): - if frame.f_code.co_filename.count('rpc.py'): - return True - else: - prev_frame = frame.f_back - prev_name = prev_frame.f_code.co_filename - if 'idlelib' in prev_name and 'debugger' in prev_name: - # catch both idlelib/debugger.py and idlelib/debugger_r.py - # on both Posix and Windows - return False - return self.in_rpc_code(prev_frame) - - def __frame2message(self, frame): - code = frame.f_code - filename = code.co_filename - lineno = frame.f_lineno - basename = os.path.basename(filename) - message = f"{basename}:{lineno}" - if code.co_name != "?": - message = f"{message}: {code.co_name}()" - return message + message = _frame2message(frame) + self.gui.interaction(message, frame, exc_info) + +def _in_rpc_code(frame): + "Determine if debugger is within RPC code." + if frame.f_code.co_filename.count('rpc.py'): + return True # Skip this frame. + else: + prev_frame = frame.f_back + if prev_frame is None: + return False + prev_name = prev_frame.f_code.co_filename + if 'idlelib' in prev_name and 'debugger' in prev_name: + # catch both idlelib/debugger.py and idlelib/debugger_r.py + # on both Posix and Windows + return False + return _in_rpc_code(prev_frame) + +def _frame2message(frame): + """Return a message string for frame.""" + code = frame.f_code + filename = code.co_filename + lineno = frame.f_lineno + basename = os.path.basename(filename) + message = f"{basename}:{lineno}" + if code.co_name != "?": + message = f"{message}: {code.co_name}()" + return message class Debugger: - - vstack = vsource = vlocals = vglobals = None + """The debugger interface. + + This class handles the drawing of the debugger window and + the interactions with the underlying debugger session. + """ + vstack = None + vsource = None + vlocals = None + vglobals = None + stackviewer = None + localsviewer = None + globalsviewer = None def __init__(self, pyshell, idb=None): + """Instantiate and draw a debugger window. + + :param pyshell: An instance of the PyShell Window + :type pyshell: :class:`idlelib.pyshell.PyShell` + + :param idb: An instance of the IDLE debugger (optional) + :type idb: :class:`idlelib.debugger.Idb` + """ if idb is None: idb = Idb(self) self.pyshell = pyshell self.idb = idb # If passed, a proxy of remote instance. self.frame = None self.make_gui() - self.interacting = 0 + self.interacting = False self.nesting_level = 0 def run(self, *args): + """Run the debugger.""" # Deal with the scenario where we've already got a program running # in the debugger and we want to start another. If that is the case, # our second 'run' was invoked from an event dispatched not from @@ -104,12 +150,13 @@ def run(self, *args): self.root.after(100, lambda: self.run(*args)) return try: - self.interacting = 1 + self.interacting = True return self.idb.run(*args) finally: - self.interacting = 0 + self.interacting = False def close(self, event=None): + """Close the debugger and window.""" try: self.quit() except Exception: @@ -127,6 +174,7 @@ def close(self, event=None): self.top.destroy() def make_gui(self): + """Draw the debugger gui on the screen.""" pyshell = self.pyshell self.flist = pyshell.flist self.root = root = pyshell.root @@ -135,11 +183,11 @@ def make_gui(self): self.top.wm_iconname("Debug") top.wm_protocol("WM_DELETE_WINDOW", self.close) self.top.bind("", self.close) - # + self.bframe = bframe = Frame(top) self.bframe.pack(anchor="w") self.buttons = bl = [] - # + self.bcont = b = Button(bframe, text="Go", command=self.cont) bl.append(b) self.bstep = b = Button(bframe, text="Step", command=self.step) @@ -150,14 +198,14 @@ def make_gui(self): bl.append(b) self.bret = b = Button(bframe, text="Quit", command=self.quit) bl.append(b) - # + for b in bl: b.configure(state="disabled") b.pack(side="left") - # + self.cframe = cframe = Frame(bframe) self.cframe.pack(side="left") - # + if not self.vstack: self.__class__.vstack = BooleanVar(top) self.vstack.set(1) @@ -180,20 +228,20 @@ def make_gui(self): self.bglobals = Checkbutton(cframe, text="Globals", command=self.show_globals, variable=self.vglobals) self.bglobals.grid(row=1, column=1) - # + self.status = Label(top, anchor="w") self.status.pack(anchor="w") self.error = Label(top, anchor="w") self.error.pack(anchor="w", fill="x") self.errorbg = self.error.cget("background") - # + self.fstack = Frame(top, height=1) self.fstack.pack(expand=1, fill="both") self.flocals = Frame(top) self.flocals.pack(expand=1, fill="both") self.fglobals = Frame(top, height=1) self.fglobals.pack(expand=1, fill="both") - # + if self.vstack.get(): self.show_stack() if self.vlocals.get(): @@ -204,7 +252,7 @@ def make_gui(self): def interaction(self, message, frame, info=None): self.frame = frame self.status.configure(text=message) - # + if info: type, value, tb = info try: @@ -223,28 +271,28 @@ def interaction(self, message, frame, info=None): tb = None bg = self.errorbg self.error.configure(text=m1, background=bg) - # + sv = self.stackviewer if sv: stack, i = self.idb.get_stack(self.frame, tb) sv.load_stack(stack, i) - # + self.show_variables(1) - # + if self.vsource.get(): self.sync_source_line() - # + for b in self.buttons: b.configure(state="normal") - # + self.top.wakeup() # Nested main loop: Tkinter's main loop is not reentrant, so use # Tcl's vwait facility, which reenters the event loop until an - # event handler sets the variable we're waiting on + # event handler sets the variable we're waiting on. self.nesting_level += 1 self.root.tk.call('vwait', '::idledebugwait') self.nesting_level -= 1 - # + for b in self.buttons: b.configure(state="disabled") self.status.configure(text="") @@ -288,8 +336,6 @@ def quit(self): def abort_loop(self): self.root.tk.call('set', '::idledebugwait', '1') - stackviewer = None - def show_stack(self): if not self.stackviewer and self.vstack.get(): self.stackviewer = sv = StackViewer(self.fstack, self.flist, self) @@ -311,9 +357,6 @@ def show_frame(self, stackitem): self.frame = stackitem[0] # lineno is stackitem[1] self.show_variables() - localsviewer = None - globalsviewer = None - def show_locals(self): lv = self.localsviewer if self.vlocals.get(): @@ -354,26 +397,32 @@ def show_variables(self, force=0): if gv: gv.load_dict(gdict, force, self.pyshell.interp.rpcclt) - def set_breakpoint_here(self, filename, lineno): + def set_breakpoint(self, filename, lineno): + """Set a filename-lineno breakpoint in the debugger. + + Called from self.load_breakpoints and EW.setbreakpoint + """ self.idb.set_break(filename, lineno) - def clear_breakpoint_here(self, filename, lineno): + def clear_breakpoint(self, filename, lineno): self.idb.clear_break(filename, lineno) def clear_file_breaks(self, filename): self.idb.clear_all_file_breaks(filename) def load_breakpoints(self): - "Load PyShellEditorWindow breakpoints into subprocess debugger" + """Load PyShellEditorWindow breakpoints into subprocess debugger.""" for editwin in self.pyshell.flist.inversedict: filename = editwin.io.filename try: for lineno in editwin.breakpoints: - self.set_breakpoint_here(filename, lineno) + self.set_breakpoint(filename, lineno) except AttributeError: continue + class StackViewer(ScrolledList): + "Code stack viewer for debugger GUI." def __init__(self, master, flist, gui): if macosx.isAquaTk(): @@ -414,12 +463,12 @@ def load_stack(self, stack, index=None): self.select(index) def popup_event(self, event): - "override base method" + "Override base method." if self.stack: return ScrolledList.popup_event(self, event) def fill_menu(self): - "override base method" + "Override base method." menu = self.menu menu.add_command(label="Go to source line", command=self.goto_source_line) @@ -427,12 +476,12 @@ def fill_menu(self): command=self.show_stack_frame) def on_select(self, index): - "override base method" + "Override base method." if 0 <= index < len(self.stack): self.gui.show_frame(self.stack[index]) def on_double(self, index): - "override base method" + "Override base method." self.show_source(index) def goto_source_line(self): @@ -457,6 +506,7 @@ def show_source(self, index): class NamespaceViewer: + "Global/local namespace viewer for debugger GUI." def __init__(self, master, title, dict=None): width = 0 @@ -509,7 +559,7 @@ def load_dict(self, dict, force=0, rpc_client=None): # There is also an obscure bug in sorted(dict) where the # interpreter gets into a loop requesting non-existing dict[0], # dict[1], dict[2], etc from the debugger_r.DictProxy. - ### + # TODO recheck above; see debugger_r 159ff, debugobj 60. keys_list = dict.keys() names = sorted(keys_list) ### @@ -544,6 +594,7 @@ def load_dict(self, dict, force=0, rpc_client=None): def close(self): self.frame.destroy() + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_debugger', verbosity=2, exit=False) diff --git a/Lib/idlelib/debugobj.py b/Lib/idlelib/debugobj.py index 71d01c7070df54..156377f8ed26ac 100644 --- a/Lib/idlelib/debugobj.py +++ b/Lib/idlelib/debugobj.py @@ -93,7 +93,8 @@ def setfunction(value, key=key, object=self.object): class DictTreeItem(SequenceTreeItem): def keys(self): - keys = list(self.object.keys()) + # TODO return sorted(self.object) + keys = list(self.object) try: keys.sort() except: @@ -119,7 +120,7 @@ def make_objecttreeitem(labeltext, object, setfunction=None): return c(labeltext, object, setfunction) -def _object_browser(parent): # htest # +def _debug_object_browser(parent): # htest # import sys from tkinter import Toplevel top = Toplevel(parent) @@ -134,9 +135,10 @@ def _object_browser(parent): # htest # node = TreeNode(sc.canvas, None, item) node.update() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_debugobj', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_object_browser) + run(_debug_object_browser) diff --git a/Lib/idlelib/delegator.py b/Lib/idlelib/delegator.py index 55c95da8532f47..93ae8bbd43ff44 100644 --- a/Lib/idlelib/delegator.py +++ b/Lib/idlelib/delegator.py @@ -28,6 +28,7 @@ def setdelegate(self, delegate): self.resetcache() self.delegate = delegate + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_delegator', verbosity=2) diff --git a/Lib/idlelib/dynoption.py b/Lib/idlelib/dynoption.py index d5dfc3eda13f60..b8937f7106ca75 100644 --- a/Lib/idlelib/dynoption.py +++ b/Lib/idlelib/dynoption.py @@ -29,6 +29,7 @@ def SetMenu(self,valueList,value=None): if value: self.variable.set(value) + def _dyn_option_menu(parent): # htest # from tkinter import Toplevel # + StringVar, Button @@ -49,6 +50,7 @@ def update(): button = Button(top, text="Change option set", command=update) button.pack() + if __name__ == '__main__': # Only module without unittests because of intention to replace. from idlelib.idle_test.htest import run diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 21402ad7139173..6ad383f460c7ee 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1571,7 +1571,7 @@ def reindent_to(self, column): # blocks are found). def guess_indent(self): - opener, indented = IndentSearcher(self.text, self.tabwidth).run() + opener, indented = IndentSearcher(self.text).run() if opener and indented: raw, indentsmall = get_line_indent(opener, self.tabwidth) raw, indentlarge = get_line_indent(indented, self.tabwidth) @@ -1609,15 +1609,10 @@ def get_line_indent(line, tabwidth): class IndentSearcher: + "Manage initial indent guess, returned by run method." - # .run() chews over the Text widget, looking for a block opener - # and the stmt following it. Returns a pair, - # (line containing block opener, line containing stmt) - # Either or both may be None. - - def __init__(self, text, tabwidth): + def __init__(self, text): self.text = text - self.tabwidth = tabwidth self.i = self.finished = 0 self.blkopenline = self.indentedline = None @@ -1633,7 +1628,8 @@ def readline(self): def tokeneater(self, type, token, start, end, line, INDENT=tokenize.INDENT, NAME=tokenize.NAME, - OPENERS=('class', 'def', 'for', 'if', 'try', 'while')): + OPENERS=('class', 'def', 'for', 'if', 'match', 'try', + 'while', 'with')): if self.finished: pass elif type == NAME and token in OPENERS: @@ -1643,23 +1639,22 @@ def tokeneater(self, type, token, start, end, line, self.finished = 1 def run(self): - save_tabsize = tokenize.tabsize - tokenize.tabsize = self.tabwidth + """Return 2 lines containing block opener and and indent. + + Either the indent line or both may be None. + """ try: - try: - tokens = tokenize.generate_tokens(self.readline) - for token in tokens: - self.tokeneater(*token) - except (tokenize.TokenError, SyntaxError): - # since we cut off the tokenizer early, we can trigger - # spurious errors - pass - finally: - tokenize.tabsize = save_tabsize + tokens = tokenize.generate_tokens(self.readline) + for token in tokens: + self.tokeneater(*token) + except (tokenize.TokenError, SyntaxError): + # Stopping the tokenizer early can trigger spurious errors. + pass return self.blkopenline, self.indentedline ### end autoindent code ### + def prepstr(s): """Extract the underscore from a string. @@ -1753,6 +1748,7 @@ def _editor_window(parent): # htest # # Does not stop error, neither does following # edit.text.bind("<>", edit.close_event) + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_editor', verbosity=2, exit=False) diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index f87781d2570fe0..e27e5d32a0ff63 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -124,6 +124,7 @@ def _test(): # TODO check and convert to htest if flist.inversedict: root.mainloop() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_filelist', verbosity=2) diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index 12513594b76f8f..ef14349960bfa2 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -204,15 +204,17 @@ def _grep_dialog(parent): # htest # frame.pack() text = Text(frame, height=5) text.pack() + text.insert('1.0', 'import grep') def show_grep_dialog(): - text.tag_add(SEL, "1.0", END) + text.tag_add(SEL, "1.0", '1.end') grep(text, flist=flist) - text.tag_remove(SEL, "1.0", END) + text.tag_remove(SEL, "1.0", '1.end') button = Button(frame, text="Show GrepDialog", command=show_grep_dialog) button.pack() + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_grep', verbosity=2, exit=False) diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index cc027b9cef4f5b..3cc7e36e35555b 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -278,7 +278,8 @@ def copy_strip(): out.write(line.rstrip() + b'\n') print(f'{src} copied to {dst}') -def show_idlehelp(parent): + +def _helpwindow(parent): "Create HelpWindow; called from Idle Help event handler." filename = join(abspath(dirname(__file__)), 'help.html') if not isfile(filename): @@ -286,9 +287,10 @@ def show_idlehelp(parent): return HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version()) + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_help', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(show_idlehelp) + run(_helpwindow) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index a0085a40b980ef..cfa4ca781f087d 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -11,15 +11,12 @@ from idlelib import textview -version = python_version() +pyver = python_version() - -def build_bits(): - "Return bits for platform." - if sys.platform == 'darwin': - return '64' if sys.maxsize > 2**32 else '32' - else: - return architecture()[0][:2] +if sys.platform == 'darwin': + bits = '64' if sys.maxsize > 2**32 else '32' +else: + bits = architecture()[0][:2] class AboutDialog(Toplevel): @@ -45,7 +42,7 @@ def __init__(self, parent, title=None, *, _htest=False, _utest=False): self.create_widgets() self.resizable(height=False, width=False) self.title(title or - f'About IDLE {version} ({build_bits()} bit)') + f'About IDLE {pyver} ({bits} bit)') self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.ok) @@ -76,8 +73,8 @@ def create_widgets(self): bg=self.bg, font=('courier', 24, 'bold')) header.grid(row=0, column=0, sticky=E, padx=10, pady=10) - tk_patchlevel = self.info_patchlevel() - ext = '.png' if tk_patchlevel >= (8, 6) else '.gif' + tkpatch = self._root().getvar('tk_patchLevel') + ext = '.png' if tkpatch >= '8.6' else '.gif' icon = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'Icons', f'idle_48{ext}') self.icon_image = PhotoImage(master=self._root(), file=icon) @@ -102,13 +99,11 @@ def create_widgets(self): height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - pyver = Label(frame_background, - text='Python version: ' + version, - fg=self.fg, bg=self.bg) - pyver.grid(row=9, column=0, sticky=W, padx=10, pady=0) - tkver = Label(frame_background, text=f'Tk version: {tk_patchlevel}', - fg=self.fg, bg=self.bg) - tkver.grid(row=9, column=1, sticky=W, padx=2, pady=0) + tclver = str(self.info_patchlevel()) + tkver = ' and ' + tkpatch if tkpatch != tclver else '' + versions = f"Python {pyver} with tcl/tk {tclver}{tkver}" + vers = Label(frame_background, text=versions, fg=self.fg, bg=self.bg) + vers.grid(row=9, column=0, sticky=W, padx=10, pady=0) py_buttons = Frame(frame_background, bg=self.bg) py_buttons.grid(row=10, column=0, columnspan=2, sticky=NSEW) self.py_license = Button(py_buttons, text='License', width=8, @@ -128,10 +123,10 @@ def create_widgets(self): height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - idlever = Label(frame_background, - text='IDLE version: ' + version, + idle = Label(frame_background, + text='IDLE', fg=self.fg, bg=self.bg) - idlever.grid(row=12, column=0, sticky=W, padx=10, pady=0) + idle.grid(row=12, column=0, sticky=W, padx=10, pady=0) idle_buttons = Frame(frame_background, bg=self.bg) idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) self.readme = Button(idle_buttons, text='README', width=8, diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index d297f8aa0094ee..997f85ff5a78b2 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -1,38 +1,36 @@ -'''Run human tests of Idle's window, dialog, and popup widgets. - -run(*tests) -Create a master Tk window. Within that, run each callable in tests -after finding the matching test spec in this file. If tests is empty, -run an htest for each spec dict in this file after finding the matching -callable in the module named in the spec. Close the window to skip or -end the test. - -In a tested module, let X be a global name bound to a callable (class -or function) whose .__name__ attribute is also X (the usual situation). -The first parameter of X must be 'parent'. When called, the parent -argument will be the root window. X must create a child Toplevel -window (or subclass thereof). The Toplevel may be a test widget or -dialog, in which case the callable is the corresponding class. Or the -Toplevel may contain the widget to be tested or set up a context in -which a test widget is invoked. In this latter case, the callable is a -wrapper function that sets up the Toplevel and other objects. Wrapper -function names, such as _editor_window', should start with '_'. +"""Run human tests of Idle's window, dialog, and popup widgets. + +run(*tests) Create a master Tk() htest window. Within that, run each +callable in tests after finding the matching test spec in this file. If +tests is empty, run an htest for each spec dict in this file after +finding the matching callable in the module named in the spec. Close +the master window to end testing. + +In a tested module, let X be a global name bound to a callable (class or +function) whose .__name__ attribute is also X (the usual situation). The +first parameter of X must be 'parent' or 'master'. When called, the +first argument will be the root window. X must create a child +Toplevel(parent/master) (or subclass thereof). The Toplevel may be a +test widget or dialog, in which case the callable is the corresponding +class. Or the Toplevel may contain the widget to be tested or set up a +context in which a test widget is invoked. In this latter case, the +callable is a wrapper function that sets up the Toplevel and other +objects. Wrapper function names, such as _editor_window', should start +with '_' and be lowercase. End the module with if __name__ == '__main__': - + from idlelib.idle_test.htest import run - run(X) + run(callable) # There could be multiple comma-separated callables. -To have wrapper functions and test invocation code ignored by coveragepy -reports, put '# htest #' on the def statement header line. - -def _wrapper(parent): # htest # - -Also make sure that the 'if __name__' line matches the above. Then have -make sure that .coveragerc includes the following. +To have wrapper functions ignored by coverage reports, tag the def +header like so: "def _wrapper(parent): # htest #". Use the same tag +for htest lines in widget code. Make sure that the 'if __name__' line +matches the above. Then have make sure that .coveragerc includes the +following: [report] exclude_lines = @@ -46,7 +44,7 @@ def _wrapper(parent): # htest # following template, with X.__name__ prepended to '_spec'. When all tests are run, the prefix is use to get X. -_spec = { +callable_spec = { 'file': '', 'kwds': {'title': ''}, 'msg': "" @@ -54,16 +52,16 @@ def _wrapper(parent): # htest # file (no .py): run() imports file.py. kwds: augmented with {'parent':root} and passed to X as **kwds. -title: an example kwd; some widgets need this, delete if not. +title: an example kwd; some widgets need this, delete line if not. msg: master window hints about testing the widget. -Modules and classes not being tested at the moment: -pyshell.PyShellEditorWindow -debugger.Debugger -autocomplete_w.AutoCompleteWindow -outwin.OutputWindow (indirectly being tested with grep test) -''' +TODO test these modules and classes: + autocomplete_w.AutoCompleteWindow + debugger.Debugger + outwin.OutputWindow (indirectly being tested with grep test) + pyshell.PyShellEditorWindow +""" import idlelib.pyshell # Set Windows DPI awareness before Tk(). from importlib import import_module @@ -91,15 +89,6 @@ def _wrapper(parent): # htest # "Force-open-calltip does not work here.\n" } -_module_browser_spec = { - 'file': 'browser', - 'kwds': {}, - 'msg': "Inspect names of module, class(with superclass if " - "applicable), methods and functions.\nToggle nested items.\n" - "Double clicking on items prints a traceback for an exception " - "that is ignored." - } - _color_delegator_spec = { 'file': 'colorizer', 'kwds': {}, @@ -109,16 +98,6 @@ def _wrapper(parent): # htest # "The default color scheme is in idlelib/config-highlight.def" } -CustomRun_spec = { - 'file': 'query', - 'kwds': {'title': 'Customize query.py Run', - '_htest': True}, - 'msg': "Enter with or [Run]. Print valid entry to Shell\n" - "Arguments are parsed into a list\n" - "Mode is currently restart True or False\n" - "Close dialog with valid entry, , [Cancel], [X]" - } - ConfigDialog_spec = { 'file': 'configdialog', 'kwds': {'title': 'ConfigDialogTest', @@ -135,6 +114,24 @@ def _wrapper(parent): # htest # "changes made have persisted." } +CustomRun_spec = { + 'file': 'query', + 'kwds': {'title': 'Customize query.py Run', + '_htest': True}, + 'msg': "Enter with or [OK]. Print valid entry to Shell\n" + "Arguments are parsed into a list\n" + "Mode is currently restart True or False\n" + "Close dialog with valid entry, , [Cancel], [X]" + } + +_debug_object_browser_spec = { + 'file': 'debugobj', + 'kwds': {}, + 'msg': "Double click on items up to the lowest level.\n" + "Attributes of the objects and related information " + "will be displayed side-by-side at each level." + } + # TODO Improve message _dyn_option_menu_spec = { 'file': 'dynoption', @@ -152,7 +149,7 @@ def _wrapper(parent): # htest # "Best to close editor first." } -GetKeysDialog_spec = { +GetKeysWindow_spec = { 'file': 'config_key', 'kwds': {'title': 'Test keybindings', 'action': 'find-again', @@ -173,8 +170,8 @@ def _wrapper(parent): # htest # 'msg': "Click the 'Show GrepDialog' button.\n" "Test the various 'Find-in-files' functions.\n" "The results should be displayed in a new '*Output*' window.\n" - "'Right-click'->'Go to file/line' anywhere in the search results " - "should open that file \nin a new EditorWindow." + "'Right-click'->'Go to file/line' in the search results\n " + "should open that file in a new EditorWindow." } HelpSource_spec = { @@ -190,7 +187,14 @@ def _wrapper(parent): # htest # "Any url ('www...', 'http...') is accepted.\n" "Test Browse with and without path, as cannot unittest.\n" "[Ok] or prints valid entry to shell\n" - "[Cancel] or prints None to shell" + ", [Cancel], or [X] prints None to shell" + } + +_helpwindow_spec = { + 'file': 'help', + 'kwds': {}, + 'msg': "If the help text displays, this works.\n" + "Text is selectable. Window is scrollable." } _io_binding_spec = { @@ -206,56 +210,36 @@ def _wrapper(parent): # htest # "Check that changes were saved by opening the file elsewhere." } -_linenumbers_drag_scrolling_spec = { - 'file': 'sidebar', +_multi_call_spec = { + 'file': 'multicall', 'kwds': {}, - 'msg': textwrap.dedent("""\ - 1. Click on the line numbers and drag down below the edge of the - window, moving the mouse a bit and then leaving it there for a while. - The text and line numbers should gradually scroll down, with the - selection updated continuously. - - 2. With the lines still selected, click on a line number above the - selected lines. Only the line whose number was clicked should be - selected. - - 3. Repeat step #1, dragging to above the window. The text and line - numbers should gradually scroll up, with the selection updated - continuously. - - 4. Repeat step #2, clicking a line number below the selection."""), + 'msg': "The following should trigger a print to console or IDLE Shell.\n" + "Entering and leaving the text area, key entry, ,\n" + ", , , \n" + ", and focusing elsewhere." } -_multi_call_spec = { - 'file': 'multicall', +_module_browser_spec = { + 'file': 'browser', 'kwds': {}, - 'msg': "The following actions should trigger a print to console or IDLE" - " Shell.\nEntering and leaving the text area, key entry, " - ",\n, , " - ", \n, and " - "focusing out of the window\nare sequences to be tested." + 'msg': textwrap.dedent(""" + "Inspect names of module, class(with superclass if applicable), + "methods and functions. Toggle nested items. Double clicking + "on items prints a traceback for an exception that is ignored.""") } _multistatus_bar_spec = { 'file': 'statusbar', 'kwds': {}, 'msg': "Ensure presence of multi-status bar below text area.\n" - "Click 'Update Status' to change the multi-status text" - } - -_object_browser_spec = { - 'file': 'debugobj', - 'kwds': {}, - 'msg': "Double click on items up to the lowest level.\n" - "Attributes of the objects and related information " - "will be displayed side-by-side at each level." + "Click 'Update Status' to change the status text" } -_path_browser_spec = { +PathBrowser_spec = { 'file': 'pathbrowser', - 'kwds': {}, + 'kwds': {'_htest': True}, 'msg': "Test for correct display of all paths in sys.path.\n" - "Toggle nested items up to the lowest level.\n" + "Toggle nested items out to the lowest level.\n" "Double clicking on an item prints a traceback\n" "for an exception that is ignored." } @@ -291,6 +275,15 @@ def _wrapper(parent): # htest # "Click [Close] or [X] to close the 'Replace Dialog'." } +_scrolled_list_spec = { + 'file': 'scrolledlist', + 'kwds': {}, + 'msg': "You should see a scrollable list of items\n" + "Selecting (clicking) or double clicking an item " + "prints the name to the console or Idle shell.\n" + "Right clicking an item will display a popup." + } + _search_dialog_spec = { 'file': 'search', 'kwds': {}, @@ -306,28 +299,31 @@ def _wrapper(parent): # htest # "Its only action is to close." } -_scrolled_list_spec = { - 'file': 'scrolledlist', +_sidebar_number_scrolling_spec = { + 'file': 'sidebar', 'kwds': {}, - 'msg': "You should see a scrollable list of items\n" - "Selecting (clicking) or double clicking an item " - "prints the name to the console or Idle shell.\n" - "Right clicking an item will display a popup." - } + 'msg': textwrap.dedent("""\ + 1. Click on the line numbers and drag down below the edge of the + window, moving the mouse a bit and then leaving it there for a + while. The text and line numbers should gradually scroll down, + with the selection updated continuously. -show_idlehelp_spec = { - 'file': 'help', - 'kwds': {}, - 'msg': "If the help text displays, this works.\n" - "Text is selectable. Window is scrollable." + 2. With the lines still selected, click on a line number above + or below the selected lines. Only the line whose number was + clicked should be selected. + + 3. Repeat step #1, dragging to above the window. The text and + line numbers should gradually scroll up, with the selection + updated continuously. + + 4. Repeat step #2, clicking a line number below the selection."""), } -_stack_viewer_spec = { +_stackbrowser_spec = { 'file': 'stackviewer', 'kwds': {}, 'msg': "A stacktrace for a NameError exception.\n" - "Expand 'idlelib ...' and ''.\n" - "Check that exc_value, exc_tb, and exc_type are correct.\n" + "Should have NameError and 1 traceback line." } _tooltip_spec = { @@ -370,11 +366,12 @@ def _wrapper(parent): # htest # } def run(*tests): + "Run callables in tests." root = tk.Tk() root.title('IDLE htest') root.resizable(0, 0) - # a scrollable Label like constant width text widget. + # A scrollable Label-like constant width text widget. frameLabel = tk.Frame(root, padx=10) frameLabel.pack() text = tk.Text(frameLabel, wrap='word') @@ -384,45 +381,44 @@ def run(*tests): scrollbar.pack(side='right', fill='y', expand=False) text.pack(side='left', fill='both', expand=True) - test_list = [] # List of tuples of the form (spec, callable widget) + test_list = [] # Make list of (spec, callable) tuples. if tests: for test in tests: test_spec = globals()[test.__name__ + '_spec'] test_spec['name'] = test.__name__ test_list.append((test_spec, test)) else: - for k, d in globals().items(): - if k.endswith('_spec'): - test_name = k[:-5] - test_spec = d + for key, dic in globals().items(): + if key.endswith('_spec'): + test_name = key[:-5] + test_spec = dic test_spec['name'] = test_name mod = import_module('idlelib.' + test_spec['file']) test = getattr(mod, test_name) test_list.append((test_spec, test)) + test_list.reverse() # So can pop in proper order in next_test. test_name = tk.StringVar(root) callable_object = None test_kwds = None def next_test(): - nonlocal test_name, callable_object, test_kwds if len(test_list) == 1: next_button.pack_forget() test_spec, callable_object = test_list.pop() test_kwds = test_spec['kwds'] - test_kwds['parent'] = root test_name.set('Test ' + test_spec['name']) - text.configure(state='normal') # enable text editing - text.delete('1.0','end') - text.insert("1.0",test_spec['msg']) - text.configure(state='disabled') # preserve read-only property + text['state'] = 'normal' # Enable text replacement. + text.delete('1.0', 'end') + text.insert("1.0", test_spec['msg']) + text['state'] = 'disabled' # Restore read-only property. def run_test(_=None): - widget = callable_object(**test_kwds) + widget = callable_object(root, **test_kwds) try: - print(widget.result) + print(widget.result) # Only true for query classes(?). except AttributeError: pass @@ -441,5 +437,6 @@ def close(_=None): next_test() root.mainloop() + if __name__ == '__main__': run() diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index 08ed76fe288294..6d75cf7aa67dcc 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -85,8 +85,8 @@ def test_load_nothing(self): self.assertEqual(parser.sections(), []) def test_load_file(self): - # Borrow test/cfgparser.1 from test_configparser. - config_path = findfile('cfgparser.1') + # Borrow test/configdata/cfgparser.1 from test_configparser. + config_path = findfile('cfgparser.1', subdir='configdata') parser = config.IdleConfParser(config_path) parser.Load() @@ -274,8 +274,8 @@ def test_create_config_handlers(self): conf.CreateConfigHandlers() # Check keys are equal - self.assertCountEqual(conf.defaultCfg.keys(), conf.config_types) - self.assertCountEqual(conf.userCfg.keys(), conf.config_types) + self.assertCountEqual(conf.defaultCfg, conf.config_types) + self.assertCountEqual(conf.userCfg, conf.config_types) # Check conf parser are correct type for default_parser in conf.defaultCfg.values(): @@ -294,8 +294,8 @@ def test_create_config_handlers(self): def test_load_cfg_files(self): conf = self.new_config(_utest=True) - # Borrow test/cfgparser.1 from test_configparser. - config_path = findfile('cfgparser.1') + # Borrow test/configdata/cfgparser.1 from test_configparser. + config_path = findfile('cfgparser.1', subdir='configdata') conf.defaultCfg['foo'] = config.IdleConfParser(config_path) conf.userCfg['foo'] = config.IdleUserConfParser(config_path) diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index e5d5b4013fca57..6f8518a9bb19d0 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -430,7 +430,7 @@ def test_highlight_target_text_mouse(self): def tag_to_element(elem): for element, tag in d.theme_elements.items(): - elem[tag[0]] = element + elem[tag] = element def click_it(start): x, y, dx, dy = hs.bbox(start) diff --git a/Lib/idlelib/idle_test/test_debugger.py b/Lib/idlelib/idle_test/test_debugger.py index 35efb3411c73b5..d1c9638dd5d711 100644 --- a/Lib/idlelib/idle_test/test_debugger.py +++ b/Lib/idlelib/idle_test/test_debugger.py @@ -1,16 +1,286 @@ -"Test debugger, coverage 19%" +"""Test debugger, coverage 66% + +Try to make tests pass with draft bdbx, which may replace bdb in 3.13+. +""" from idlelib import debugger -import unittest -from test.support import requires -requires('gui') +from collections import namedtuple +from textwrap import dedent from tkinter import Tk +from test.support import requires +import unittest +from unittest import mock +from unittest.mock import Mock, patch + +"""A test python script for the debug tests.""" +TEST_CODE = dedent(""" + i = 1 + i += 2 + if i == 3: + print(i) + """) + + +class MockFrame: + "Minimal mock frame." + + def __init__(self, code, lineno): + self.f_code = code + self.f_lineno = lineno + + +class IdbTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.gui = Mock() + cls.idb = debugger.Idb(cls.gui) + + # Create test and code objects to simulate a debug session. + code_obj = compile(TEST_CODE, 'idlelib/file.py', mode='exec') + frame1 = MockFrame(code_obj, 1) + frame1.f_back = None + frame2 = MockFrame(code_obj, 2) + frame2.f_back = frame1 + cls.frame = frame2 + cls.msg = 'file.py:2: ()' + + def test_init(self): + self.assertIs(self.idb.gui, self.gui) + # Won't test super call since two Bdbs are very different. + + def test_user_line(self): + # Test that .user_line() creates a string message for a frame. + self.gui.interaction = Mock() + self.idb.user_line(self.frame) + self.gui.interaction.assert_called_once_with(self.msg, self.frame) + + def test_user_exception(self): + # Test that .user_exception() creates a string message for a frame. + exc_info = (type(ValueError), ValueError(), None) + self.gui.interaction = Mock() + self.idb.user_exception(self.frame, exc_info) + self.gui.interaction.assert_called_once_with( + self.msg, self.frame, exc_info) + + +class FunctionTest(unittest.TestCase): + # Test module functions together. + + def test_functions(self): + rpc_obj = compile(TEST_CODE,'rpc.py', mode='exec') + rpc_frame = MockFrame(rpc_obj, 2) + rpc_frame.f_back = rpc_frame + self.assertTrue(debugger._in_rpc_code(rpc_frame)) + self.assertEqual(debugger._frame2message(rpc_frame), + 'rpc.py:2: ()') + + code_obj = compile(TEST_CODE, 'idlelib/debugger.py', mode='exec') + code_frame = MockFrame(code_obj, 1) + code_frame.f_back = None + self.assertFalse(debugger._in_rpc_code(code_frame)) + self.assertEqual(debugger._frame2message(code_frame), + 'debugger.py:1: ()') + + code_frame.f_back = code_frame + self.assertFalse(debugger._in_rpc_code(code_frame)) + code_frame.f_back = rpc_frame + self.assertTrue(debugger._in_rpc_code(code_frame)) + + +class DebuggerTest(unittest.TestCase): + "Tests for Debugger that do not need a real root." + + @classmethod + def setUpClass(cls): + cls.pyshell = Mock() + cls.pyshell.root = Mock() + cls.idb = Mock() + with patch.object(debugger.Debugger, 'make_gui'): + cls.debugger = debugger.Debugger(cls.pyshell, cls.idb) + cls.debugger.root = Mock() + + def test_cont(self): + self.debugger.cont() + self.idb.set_continue.assert_called_once() + + def test_step(self): + self.debugger.step() + self.idb.set_step.assert_called_once() + + def test_quit(self): + self.debugger.quit() + self.idb.set_quit.assert_called_once() + + def test_next(self): + with patch.object(self.debugger, 'frame') as frame: + self.debugger.next() + self.idb.set_next.assert_called_once_with(frame) + + def test_ret(self): + with patch.object(self.debugger, 'frame') as frame: + self.debugger.ret() + self.idb.set_return.assert_called_once_with(frame) + + def test_clear_breakpoint(self): + self.debugger.clear_breakpoint('test.py', 4) + self.idb.clear_break.assert_called_once_with('test.py', 4) + + def test_clear_file_breaks(self): + self.debugger.clear_file_breaks('test.py') + self.idb.clear_all_file_breaks.assert_called_once_with('test.py') + + def test_set_load_breakpoints(self): + # Test the .load_breakpoints() method calls idb. + FileIO = namedtuple('FileIO', 'filename') + + class MockEditWindow(object): + def __init__(self, fn, breakpoints): + self.io = FileIO(fn) + self.breakpoints = breakpoints + + self.pyshell.flist = Mock() + self.pyshell.flist.inversedict = ( + MockEditWindow('test1.py', [4, 4]), + MockEditWindow('test2.py', [13, 44, 45]), + ) + self.debugger.set_breakpoint('test0.py', 1) + self.idb.set_break.assert_called_once_with('test0.py', 1) + self.debugger.load_breakpoints() # Call set_breakpoint 5 times. + self.idb.set_break.assert_has_calls( + [mock.call('test0.py', 1), + mock.call('test1.py', 4), + mock.call('test1.py', 4), + mock.call('test2.py', 13), + mock.call('test2.py', 44), + mock.call('test2.py', 45)]) + + def test_sync_source_line(self): + # Test that .sync_source_line() will set the flist.gotofileline with fixed frame. + test_code = compile(TEST_CODE, 'test_sync.py', 'exec') + test_frame = MockFrame(test_code, 1) + self.debugger.frame = test_frame + + self.debugger.flist = Mock() + with patch('idlelib.debugger.os.path.exists', return_value=True): + self.debugger.sync_source_line() + self.debugger.flist.gotofileline.assert_called_once_with('test_sync.py', 1) + + +class DebuggerGuiTest(unittest.TestCase): + """Tests for debugger.Debugger that need tk root. + + close needs debugger.top set in make_gui. + """ + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = root = Tk() + root.withdraw() + cls.pyshell = Mock() + cls.pyshell.root = root + cls.idb = Mock() +# stack tests fail with debugger here. +## cls.debugger = debugger.Debugger(cls.pyshell, cls.idb) +## cls.debugger.root = root +## # real root needed for real make_gui +## # run, interacting, abort_loop + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def setUp(self): + self.debugger = debugger.Debugger(self.pyshell, self.idb) + self.debugger.root = self.root + # real root needed for real make_gui + # run, interacting, abort_loop + + def test_run_debugger(self): + self.debugger.run(1, 'two') + self.idb.run.assert_called_once_with(1, 'two') + self.assertEqual(self.debugger.interacting, 0) + + def test_close(self): + # Test closing the window in an idle state. + self.debugger.close() + self.pyshell.close_debugger.assert_called_once() + + def test_show_stack(self): + self.debugger.show_stack() + self.assertEqual(self.debugger.stackviewer.gui, self.debugger) + + def test_show_stack_with_frame(self): + test_frame = MockFrame(None, None) + self.debugger.frame = test_frame + + # Reset the stackviewer to force it to be recreated. + self.debugger.stackviewer = None + self.idb.get_stack.return_value = ([], 0) + self.debugger.show_stack() + + # Check that the newly created stackviewer has the test gui as a field. + self.assertEqual(self.debugger.stackviewer.gui, self.debugger) + self.idb.get_stack.assert_called_once_with(test_frame, None) + + +class StackViewerTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def setUp(self): + self.code = compile(TEST_CODE, 'test_stackviewer.py', 'exec') + self.stack = [ + (MockFrame(self.code, 1), 1), + (MockFrame(self.code, 2), 2) + ] + # Create a stackviewer and load the test stack. + self.sv = debugger.StackViewer(self.root, None, None) + self.sv.load_stack(self.stack) + + def test_init(self): + # Test creation of StackViewer. + gui = None + flist = None + master_window = self.root + sv = debugger.StackViewer(master_window, flist, gui) + self.assertTrue(hasattr(sv, 'stack')) + + def test_load_stack(self): + # Test the .load_stack() method against a fixed test stack. + # Check the test stack is assigned and the list contains the repr of them. + self.assertEqual(self.sv.stack, self.stack) + self.assertTrue('?.(), line 1:' in self.sv.get(0)) + self.assertEqual(self.sv.get(1), '?.(), line 2: ') + + def test_show_source(self): + # Test the .show_source() method against a fixed test stack. + # Patch out the file list to monitor it + self.sv.flist = Mock() + # Patch out isfile to pretend file exists. + with patch('idlelib.debugger.os.path.isfile', return_value=True) as isfile: + self.sv.show_source(1) + isfile.assert_called_once_with('test_stackviewer.py') + self.sv.flist.open.assert_called_once_with('test_stackviewer.py') + class NameSpaceTest(unittest.TestCase): @classmethod def setUpClass(cls): + requires('gui') cls.root = Tk() cls.root.withdraw() @@ -23,7 +293,5 @@ def test_init(self): debugger.NamespaceViewer(self.root, 'Test') -# Other classes are Idb, Debugger, and StackViewer. - if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_debugobj.py b/Lib/idlelib/idle_test/test_debugobj.py index 131ce22b8bb69b..90ace4e1bc4f9e 100644 --- a/Lib/idlelib/idle_test/test_debugobj.py +++ b/Lib/idlelib/idle_test/test_debugobj.py @@ -37,7 +37,7 @@ def test_isexpandable(self): def test_keys(self): ti = debugobj.SequenceTreeItem('label', 'abc') - self.assertEqual(list(ti.keys()), [0, 1, 2]) + self.assertEqual(list(ti.keys()), [0, 1, 2]) # keys() is a range. class DictTreeItemTest(unittest.TestCase): @@ -50,7 +50,7 @@ def test_isexpandable(self): def test_keys(self): ti = debugobj.DictTreeItem('label', {1:1, 0:0, 2:2}) - self.assertEqual(ti.keys(), [0, 1, 2]) + self.assertEqual(ti.keys(), [0, 1, 2]) # keys() is a sorted list. if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index fdb47abf43fb77..9296a6d235fbbe 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -1,10 +1,10 @@ -"Test editor, coverage 35%." +"Test editor, coverage 53%." from idlelib import editor import unittest from collections import namedtuple from test.support import requires -from tkinter import Tk +from tkinter import Tk, Text Editor = editor.EditorWindow @@ -31,7 +31,7 @@ def test_init(self): e._close() -class TestGetLineIndent(unittest.TestCase): +class GetLineIndentTest(unittest.TestCase): def test_empty_lines(self): for tabwidth in [1, 2, 4, 6, 8]: for line in ['', '\n']: @@ -181,6 +181,36 @@ def test_indent_and_newline_event(self): eq(get('1.0', 'end'), ' def f1(self, a,\n \n return a + b\n') +class IndentSearcherTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def test_searcher(self): + text = self.text + searcher = (self.text) + test_info = (# text, (block, indent)) + ("", (None, None)), + ("[1,", (None, None)), # TokenError + ("if 1:\n", ('if 1:\n', None)), + ("if 1:\n 2\n 3\n", ('if 1:\n', ' 2\n')), + ) + for code, expected_pair in test_info: + with self.subTest(code=code): + insert(text, code) + actual_pair = editor.IndentSearcher(text).run() + self.assertEqual(actual_pair, expected_pair) + + class RMenuTest(unittest.TestCase): @classmethod diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index b915535acac0cc..8b79487b15d4cd 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -36,7 +36,7 @@ def tearDownClass(cls): del cls.root def test_build_bits(self): - self.assertIn(help_about.build_bits(), ('32', '64')) + self.assertIn(help_about.bits, ('32', '64')) def test_dialog_title(self): """Test about dialog title""" @@ -107,7 +107,7 @@ def test_dialog_title(self): """Test about dialog title""" self.assertEqual(self.dialog.title(), f'About IDLE {python_version()}' - f' ({help_about.build_bits()} bit)') + f' ({help_about.bits} bit)') class CloseTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_iomenu.py b/Lib/idlelib/idle_test/test_iomenu.py index 2fb836dba21672..e0642cf0cabef0 100644 --- a/Lib/idlelib/idle_test/test_iomenu.py +++ b/Lib/idlelib/idle_test/test_iomenu.py @@ -8,6 +8,12 @@ from idlelib import util from idlelib.idle_test.mock_idle import Func +# Fail if either tokenize.open and t.detect_encoding does not exist. +# These are used in loadfile and encode. +# Also used in pyshell.MI.execfile and runscript.tabnanny. +from tokenize import open, detect_encoding +# Remove when we have proper tests that use both. + class IOBindingTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_stackviewer.py b/Lib/idlelib/idle_test/test_stackviewer.py index 98f53f9537bb25..55f510382bf4c3 100644 --- a/Lib/idlelib/idle_test/test_stackviewer.py +++ b/Lib/idlelib/idle_test/test_stackviewer.py @@ -6,19 +6,12 @@ from tkinter import Tk from idlelib.tree import TreeNode, ScrolledCanvas -import sys class StackBrowserTest(unittest.TestCase): @classmethod def setUpClass(cls): - svs = stackviewer.sys - try: - abc - except NameError: - svs.last_type, svs.last_value, svs.last_traceback = ( - sys.exc_info()) requires('gui') cls.root = Tk() @@ -26,8 +19,6 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - svs = stackviewer.sys - del svs.last_traceback, svs.last_type, svs.last_value cls.root.update_idletasks() ## for id in cls.root.tk.call('after', 'info'): @@ -36,7 +27,10 @@ def tearDownClass(cls): del cls.root def test_init(self): - sb = stackviewer.StackBrowser(self.root) + try: + abc + except NameError as exc: + sb = stackviewer.StackBrowser(self.root, exc) isi = self.assertIsInstance isi(stackviewer.sc, ScrolledCanvas) isi(stackviewer.item, stackviewer.StackTreeItem) diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index af8159c2b33f51..667623ec71ac98 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -393,13 +393,15 @@ def updaterecentfileslist(self,filename): if self.editwin.flist: self.editwin.update_recent_files_list(filename) + def _io_binding(parent): # htest # from tkinter import Toplevel, Text - root = Toplevel(parent) - root.title("Test IOBinding") + top = Toplevel(parent) + top.title("Test IOBinding") x, y = map(int, parent.geometry().split('+')[1:]) - root.geometry("+%d+%d" % (x, y + 175)) + top.geometry("+%d+%d" % (x, y + 175)) + class MyEditWin: def __init__(self, text): self.text = text @@ -423,12 +425,13 @@ def saveas(self, event): def savecopy(self, event): self.text.event_generate("<>") - text = Text(root) + text = Text(top) text.pack() text.focus_set() editwin = MyEditWin(text) IOBinding(editwin) + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_iomenu', verbosity=2, exit=False) diff --git a/Lib/idlelib/multicall.py b/Lib/idlelib/multicall.py index 0200f445cc9340..41f81813113062 100644 --- a/Lib/idlelib/multicall.py +++ b/Lib/idlelib/multicall.py @@ -421,6 +421,8 @@ def _multi_call(parent): # htest # top.geometry("+%d+%d" % (x, y + 175)) text = MultiCallCreator(tkinter.Text)(top) text.pack() + text.focus_set() + def bindseq(seq, n=[0]): def handler(event): print(seq) @@ -440,6 +442,7 @@ def handler(event): bindseq("") bindseq("") + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_mainmenu', verbosity=2, exit=False) diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 610031e26f1dff..5ed3f35a7af655 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -182,6 +182,7 @@ def setup(self): text.tag_raise('sel') self.write = self.owin.write + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_outwin', verbosity=2, exit=False) diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index 6de242d0000bed..48a77875ba5801 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -99,13 +99,9 @@ def listmodules(self, allnames): return sorted -def _path_browser(parent): # htest # - PathBrowser(parent, _htest=True) - parent.mainloop() - if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_pathbrowser', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_path_browser) + run(PathBrowser) diff --git a/Lib/idlelib/percolator.py b/Lib/idlelib/percolator.py index 1fe34d29f54eb2..aa73427c4915c8 100644 --- a/Lib/idlelib/percolator.py +++ b/Lib/idlelib/percolator.py @@ -86,11 +86,11 @@ def delete(self, *args): print(self.name, ": delete", args) self.delegate.delete(*args) - box = tk.Toplevel(parent) - box.title("Test Percolator") + top = tk.Toplevel(parent) + top.title("Test Percolator") x, y = map(int, parent.geometry().split('+')[1:]) - box.geometry("+%d+%d" % (x, y + 175)) - text = tk.Text(box) + top.geometry("+%d+%d" % (x, y + 175)) + text = tk.Text(top) p = Percolator(text) pin = p.insertfilter pout = p.removefilter @@ -103,13 +103,15 @@ def toggle2(): (pin if var2.get() else pout)(t2) text.pack() + text.focus_set() var1 = tk.IntVar(parent) - cb1 = tk.Checkbutton(box, text="Tracer1", command=toggle1, variable=var1) + cb1 = tk.Checkbutton(top, text="Tracer1", command=toggle1, variable=var1) cb1.pack() var2 = tk.IntVar(parent) - cb2 = tk.Checkbutton(box, text="Tracer2", command=toggle2, variable=var2) + cb2 = tk.Checkbutton(top, text="Tracer2", command=toggle2, variable=var2) cb2.pack() + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_percolator', verbosity=2, exit=False) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index bdde156166171b..1524fccd5d20f8 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -133,8 +133,8 @@ class PyShellEditorWindow(EditorWindow): def __init__(self, *args): self.breakpoints = [] EditorWindow.__init__(self, *args) - self.text.bind("<>", self.set_breakpoint_here) - self.text.bind("<>", self.clear_breakpoint_here) + self.text.bind("<>", self.set_breakpoint_event) + self.text.bind("<>", self.clear_breakpoint_event) self.text.bind("<>", self.flist.open_shell) #TODO: don't read/write this from/to .idlerc when testing @@ -155,8 +155,8 @@ def filename_changed_hook(old_hook=self.io.filename_change_hook, ("Copy", "<>", "rmenu_check_copy"), ("Paste", "<>", "rmenu_check_paste"), (None, None, None), - ("Set Breakpoint", "<>", None), - ("Clear Breakpoint", "<>", None) + ("Set Breakpoint", "<>", None), + ("Clear Breakpoint", "<>", None) ] def color_breakpoint_text(self, color=True): @@ -181,11 +181,11 @@ def set_breakpoint(self, lineno): self.breakpoints.append(lineno) try: # update the subprocess debugger debug = self.flist.pyshell.interp.debugger - debug.set_breakpoint_here(filename, lineno) + debug.set_breakpoint(filename, lineno) except: # but debugger may not be active right now.... pass - def set_breakpoint_here(self, event=None): + def set_breakpoint_event(self, event=None): text = self.text filename = self.io.filename if not filename: @@ -194,7 +194,7 @@ def set_breakpoint_here(self, event=None): lineno = int(float(text.index("insert"))) self.set_breakpoint(lineno) - def clear_breakpoint_here(self, event=None): + def clear_breakpoint_event(self, event=None): text = self.text filename = self.io.filename if not filename: @@ -209,7 +209,7 @@ def clear_breakpoint_here(self, event=None): "insert lineend +1char") try: debug = self.flist.pyshell.interp.debugger - debug.clear_breakpoint_here(filename, lineno) + debug.clear_breakpoint(filename, lineno) except: pass @@ -747,10 +747,11 @@ def showtraceback(self): self.tkconsole.open_stack_viewer() def checklinecache(self): - c = linecache.cache - for key in list(c.keys()): + "Remove keys other than ''." + cache = linecache.cache + for key in list(cache): # Iterate list because mutate cache. if key[:1] + key[-1:] != "<>": - del c[key] + del cache[key] def runcommand(self, code): "Run the code without invoking the debugger" @@ -1363,19 +1364,19 @@ def runit(self): self.text.tag_remove(self.user_input_insert_tags, index_before) self.shell_sidebar.update_sidebar() - def open_stack_viewer(self, event=None): + def open_stack_viewer(self, event=None): # -n mode only if self.interp.rpcclt: return self.interp.remote_stack_viewer() + + from idlelib.stackviewer import StackBrowser try: - sys.last_traceback + StackBrowser(self.root, sys.last_exc, self.flist) except: messagebox.showerror("No stack trace", "There is no stack trace yet.\n" - "(sys.last_traceback is not defined)", + "(sys.last_exc is not defined)", parent=self.text) - return - from idlelib.stackviewer import StackBrowser - StackBrowser(self.root, self.flist) + return None def view_restart_mark(self, event=None): self.text.see("iomark") @@ -1693,6 +1694,7 @@ def main(): root.destroy() capture_warnings(False) + if __name__ == "__main__": main() diff --git a/Lib/idlelib/query.py b/Lib/idlelib/query.py index df02f2123ab02f..57230e2aaca66d 100644 --- a/Lib/idlelib/query.py +++ b/Lib/idlelib/query.py @@ -368,7 +368,7 @@ def create_extra(self): sticky='we') def cli_args_ok(self): - "Validity check and parsing for command line arguments." + "Return command line arg list or None if error." cli_string = self.entry.get().strip() try: cli_args = shlex.split(cli_string, posix=True) diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 4928340e98df68..08728956abd900 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -164,6 +164,7 @@ def my_insert(*args): original_insert(*args) original_insert = redir.register("insert", my_insert) + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_redirector', verbosity=2, exit=False) diff --git a/Lib/idlelib/replace.py b/Lib/idlelib/replace.py index ca83173877ad1d..a29ca591427491 100644 --- a/Lib/idlelib/replace.py +++ b/Lib/idlelib/replace.py @@ -299,6 +299,7 @@ def show_replace(): button = Button(frame, text="Replace", command=show_replace) button.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_replace', verbosity=2, exit=False) diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 84792a82b0022c..53e80a9b42801f 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -140,11 +140,12 @@ def main(del_exitfunc=False): capture_warnings(True) sys.argv[:] = [""] - sockthread = threading.Thread(target=manage_socket, - name='SockThread', - args=((LOCALHOST, port),)) - sockthread.daemon = True - sockthread.start() + threading.Thread(target=manage_socket, + name='SockThread', + args=((LOCALHOST, port),), + daemon=True, + ).start() + while True: try: if exit_now: @@ -622,7 +623,7 @@ def get_the_completion_list(self, what, mode): def stackviewer(self, flist_oid=None): if self.user_exc_info: - typ, val, tb = self.user_exc_info + _, exc, tb = self.user_exc_info else: return None flist = None @@ -630,9 +631,8 @@ def stackviewer(self, flist_oid=None): flist = self.rpchandler.get_remote_proxy(flist_oid) while tb and tb.tb_frame.f_globals["__name__"] in ["rpc", "run"]: tb = tb.tb_next - sys.last_type = typ - sys.last_value = val - item = stackviewer.StackTreeItem(flist, tb) + exc.__traceback__ = tb + item = stackviewer.StackTreeItem(exc, flist) return debugobj_r.remote_object_tree_item(item) diff --git a/Lib/idlelib/scrolledlist.py b/Lib/idlelib/scrolledlist.py index 71fd18ab19ec8a..4fb418db326255 100644 --- a/Lib/idlelib/scrolledlist.py +++ b/Lib/idlelib/scrolledlist.py @@ -132,6 +132,7 @@ def _scrolled_list(parent): # htest # top = Toplevel(parent) x, y = map(int, parent.geometry().split('+')[1:]) top.geometry("+%d+%d" % (x+200, y + 175)) + class MyScrolledList(ScrolledList): def fill_menu(self): self.menu.add_command(label="right click") def on_select(self, index): print("select", self.get(index)) @@ -141,9 +142,10 @@ def on_double(self, index): print("double", self.get(index)) for i in range(30): scrolled_list.append("Item %02d" % i) + if __name__ == '__main__': from unittest import main - main('idlelib.idle_test.test_scrolledlist', verbosity=2,) + main('idlelib.idle_test.test_scrolledlist', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(_scrolled_list) diff --git a/Lib/idlelib/search.py b/Lib/idlelib/search.py index b35f3b59c3d2e8..935a4832257fa4 100644 --- a/Lib/idlelib/search.py +++ b/Lib/idlelib/search.py @@ -156,6 +156,7 @@ def show_find(): button = Button(frame, text="Search (selection ignored)", command=show_find) button.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_search', verbosity=2, exit=False) diff --git a/Lib/idlelib/sidebar.py b/Lib/idlelib/sidebar.py index fb1084dbf3f18b..ff77b568a786e0 100644 --- a/Lib/idlelib/sidebar.py +++ b/Lib/idlelib/sidebar.py @@ -25,10 +25,9 @@ def get_end_linenumber(text): def get_displaylines(text, index): """Display height, in lines, of a logical line in a Tk text widget.""" - res = text.count(f"{index} linestart", - f"{index} lineend", - "displaylines") - return res[0] if res else 0 + return text.count(f"{index} linestart", + f"{index} lineend", + "displaylines") def get_widget_padding(widget): """Get the total padding of a Tk widget, including its border.""" @@ -514,16 +513,16 @@ def update_colors(self): self.change_callback() -def _linenumbers_drag_scrolling(parent): # htest # +def _sidebar_number_scrolling(parent): # htest # from idlelib.idle_test.test_sidebar import Dummy_editwin - toplevel = tk.Toplevel(parent) - text_frame = tk.Frame(toplevel) + top = tk.Toplevel(parent) + text_frame = tk.Frame(top) text_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True) text_frame.rowconfigure(1, weight=1) text_frame.columnconfigure(1, weight=1) - font = idleConf.GetFont(toplevel, 'main', 'EditorWindow') + font = idleConf.GetFont(top, 'main', 'EditorWindow') text = tk.Text(text_frame, width=80, height=24, wrap=tk.NONE, font=font) text.grid(row=1, column=1, sticky=tk.NSEW) @@ -541,4 +540,4 @@ def _linenumbers_drag_scrolling(parent): # htest # main('idlelib.idle_test.test_sidebar', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_linenumbers_drag_scrolling) + run(_sidebar_number_scrolling) diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index 94ffb4eff4dd26..977c56ef15f2ae 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -1,33 +1,32 @@ +# Rename to stackbrowser or possibly consolidate with browser. + import linecache import os -import sys import tkinter as tk from idlelib.debugobj import ObjectTreeItem, make_objecttreeitem from idlelib.tree import TreeNode, TreeItem, ScrolledCanvas -def StackBrowser(root, flist=None, tb=None, top=None): +def StackBrowser(root, exc, flist=None, top=None): global sc, item, node # For testing. if top is None: top = tk.Toplevel(root) sc = ScrolledCanvas(top, bg="white", highlightthickness=0) sc.frame.pack(expand=1, fill="both") - item = StackTreeItem(flist, tb) + item = StackTreeItem(exc, flist) node = TreeNode(sc.canvas, None, item) node.expand() class StackTreeItem(TreeItem): - def __init__(self, flist=None, tb=None): + def __init__(self, exc, flist=None): self.flist = flist - self.stack = self.get_stack(tb) - self.text = self.get_exception() + self.stack = self.get_stack(None if exc is None else exc.__traceback__) + self.text = f"{type(exc).__name__}: {str(exc)}" def get_stack(self, tb): - if tb is None: - tb = sys.last_traceback stack = [] if tb and tb.tb_frame is None: tb = tb.tb_next @@ -36,17 +35,7 @@ def get_stack(self, tb): tb = tb.tb_next return stack - def get_exception(self): - type = sys.last_type - value = sys.last_value - if hasattr(type, "__name__"): - type = type.__name__ - s = str(type) - if value is not None: - s = s + ": " + str(value) - return s - - def GetText(self): + def GetText(self): # Titlecase names are overrides. return self.text def GetSubList(self): @@ -112,7 +101,7 @@ def IsExpandable(self): def GetSubList(self): sublist = [] - for key in self.object.keys(): + for key in self.object.keys(): # self.object not necessarily dict. try: value = self.object[key] except KeyError: @@ -124,7 +113,7 @@ def setfunction(value, key=key, object=self.object): return sublist -def _stack_viewer(parent): # htest # +def _stackbrowser(parent): # htest # from idlelib.pyshell import PyShellFileList top = tk.Toplevel(parent) top.title("Test StackViewer") @@ -133,23 +122,13 @@ def _stack_viewer(parent): # htest # flist = PyShellFileList(top) try: # to obtain a traceback object intentional_name_error - except NameError: - exc_type, exc_value, exc_tb = sys.exc_info() - # inject stack trace to sys - sys.last_type = exc_type - sys.last_value = exc_value - sys.last_traceback = exc_tb - - StackBrowser(top, flist=flist, top=top, tb=exc_tb) + except NameError as e: + StackBrowser(top, e, flist=flist, top=top) - # restore sys to original state - del sys.last_type - del sys.last_value - del sys.last_traceback if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_stackviewer', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_stack_viewer) + run(_stackbrowser) diff --git a/Lib/idlelib/statusbar.py b/Lib/idlelib/statusbar.py index 755fafb0ac6438..8445d4cc8dfdb9 100644 --- a/Lib/idlelib/statusbar.py +++ b/Lib/idlelib/statusbar.py @@ -26,6 +26,7 @@ def _multistatus_bar(parent): # htest # x, y = map(int, parent.geometry().split('+')[1:]) top.geometry("+%d+%d" %(x, y + 175)) top.title("Test multistatus bar") + frame = Frame(top) text = Text(frame, height=5, width=40) text.pack() @@ -42,6 +43,7 @@ def change(): button.pack(side='bottom') frame.pack() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_statusbar', verbosity=2, exit=False) diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 5f30f0f6092bfa..9c2eb47b24aec9 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -492,6 +492,7 @@ def _tree_widget(parent): # htest # node = TreeNode(sc.canvas, None, item) node.expand() + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_tree', verbosity=2, exit=False) diff --git a/Lib/idlelib/undo.py b/Lib/idlelib/undo.py index 5f10c0f05c1acb..f52446d5fcdcf8 100644 --- a/Lib/idlelib/undo.py +++ b/Lib/idlelib/undo.py @@ -339,25 +339,26 @@ def bump_depth(self, incr=1): def _undo_delegator(parent): # htest # from tkinter import Toplevel, Text, Button from idlelib.percolator import Percolator - undowin = Toplevel(parent) - undowin.title("Test UndoDelegator") + top = Toplevel(parent) + top.title("Test UndoDelegator") x, y = map(int, parent.geometry().split('+')[1:]) - undowin.geometry("+%d+%d" % (x, y + 175)) + top.geometry("+%d+%d" % (x, y + 175)) - text = Text(undowin, height=10) + text = Text(top, height=10) text.pack() text.focus_set() p = Percolator(text) d = UndoDelegator() p.insertfilter(d) - undo = Button(undowin, text="Undo", command=lambda:d.undo_event(None)) + undo = Button(top, text="Undo", command=lambda:d.undo_event(None)) undo.pack(side='left') - redo = Button(undowin, text="Redo", command=lambda:d.redo_event(None)) + redo = Button(top, text="Redo", command=lambda:d.redo_event(None)) redo.pack(side='left') - dump = Button(undowin, text="Dump", command=lambda:d.dump_event(None)) + dump = Button(top, text="Dump", command=lambda:d.dump_event(None)) dump.pack(side='left') + if __name__ == "__main__": from unittest import main main('idlelib.idle_test.test_undo', verbosity=2, exit=False) diff --git a/Lib/idlelib/util.py b/Lib/idlelib/util.py index ede670a4db5536..5ac69a7b94cb56 100644 --- a/Lib/idlelib/util.py +++ b/Lib/idlelib/util.py @@ -16,6 +16,7 @@ # .pyw is for Windows; .pyi is for stub files. py_extensions = ('.py', '.pyw', '.pyi') # Order needed for open/save dialogs. + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_util', verbosity=2) diff --git a/Lib/imghdr.py b/Lib/imghdr.py deleted file mode 100644 index 33868883470764..00000000000000 --- a/Lib/imghdr.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Recognize image file formats based on their first few bytes.""" - -from os import PathLike -import warnings - -__all__ = ["what"] - - -warnings._deprecated(__name__, remove=(3, 13)) - - -#-------------------------# -# Recognize image headers # -#-------------------------# - -def what(file, h=None): - """Return the type of image contained in a file or byte stream.""" - f = None - try: - if h is None: - if isinstance(file, (str, PathLike)): - f = open(file, 'rb') - h = f.read(32) - else: - location = file.tell() - h = file.read(32) - file.seek(location) - for tf in tests: - res = tf(h, f) - if res: - return res - finally: - if f: f.close() - return None - - -#---------------------------------# -# Subroutines per image file type # -#---------------------------------# - -tests = [] - -def test_jpeg(h, f): - """Test for JPEG data with JFIF or Exif markers; and raw JPEG.""" - if h[6:10] in (b'JFIF', b'Exif'): - return 'jpeg' - elif h[:4] == b'\xff\xd8\xff\xdb': - return 'jpeg' - -tests.append(test_jpeg) - -def test_png(h, f): - """Verify if the image is a PNG.""" - if h.startswith(b'\211PNG\r\n\032\n'): - return 'png' - -tests.append(test_png) - -def test_gif(h, f): - """Verify if the image is a GIF ('87 or '89 variants).""" - if h[:6] in (b'GIF87a', b'GIF89a'): - return 'gif' - -tests.append(test_gif) - -def test_tiff(h, f): - """Verify if the image is a TIFF (can be in Motorola or Intel byte order).""" - if h[:2] in (b'MM', b'II'): - return 'tiff' - -tests.append(test_tiff) - -def test_rgb(h, f): - """test for the SGI image library.""" - if h.startswith(b'\001\332'): - return 'rgb' - -tests.append(test_rgb) - -def test_pbm(h, f): - """Verify if the image is a PBM (portable bitmap).""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r': - return 'pbm' - -tests.append(test_pbm) - -def test_pgm(h, f): - """Verify if the image is a PGM (portable graymap).""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r': - return 'pgm' - -tests.append(test_pgm) - -def test_ppm(h, f): - """Verify if the image is a PPM (portable pixmap).""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r': - return 'ppm' - -tests.append(test_ppm) - -def test_rast(h, f): - """test for the Sun raster file.""" - if h.startswith(b'\x59\xA6\x6A\x95'): - return 'rast' - -tests.append(test_rast) - -def test_xbm(h, f): - """Verify if the image is a X bitmap (X10 or X11).""" - if h.startswith(b'#define '): - return 'xbm' - -tests.append(test_xbm) - -def test_bmp(h, f): - """Verify if the image is a BMP file.""" - if h.startswith(b'BM'): - return 'bmp' - -tests.append(test_bmp) - -def test_webp(h, f): - """Verify if the image is a WebP.""" - if h.startswith(b'RIFF') and h[8:12] == b'WEBP': - return 'webp' - -tests.append(test_webp) - -def test_exr(h, f): - """verify is the image ia a OpenEXR fileOpenEXR.""" - if h.startswith(b'\x76\x2f\x31\x01'): - return 'exr' - -tests.append(test_exr) - -#--------------------# -# Small test program # -#--------------------# - -def test(): - import sys - recursive = 0 - if sys.argv[1:] and sys.argv[1] == '-r': - del sys.argv[1:2] - recursive = 1 - try: - if sys.argv[1:]: - testall(sys.argv[1:], recursive, 1) - else: - testall(['.'], recursive, 1) - except KeyboardInterrupt: - sys.stderr.write('\n[Interrupted]\n') - sys.exit(1) - -def testall(list, recursive, toplevel): - import sys - import os - for filename in list: - if os.path.isdir(filename): - print(filename + '/:', end=' ') - if recursive or toplevel: - print('recursing down:') - import glob - names = glob.glob(os.path.join(glob.escape(filename), '*')) - testall(names, recursive, 0) - else: - print('*** directory (use -r) ***') - else: - print(filename + ':', end=' ') - sys.stdout.flush() - try: - print(what(filename)) - except OSError: - print('*** not found ***') - -if __name__ == '__main__': - test() diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 707c081cb2c5b6..f38fe5c1ab461a 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -54,8 +54,6 @@ # Fully bootstrapped at this point, import whatever you like, circular # dependencies and startup overhead minimisation permitting :) -import warnings - # Public API ######################################################### diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index c48fd506a0e4eb..d942045f3de666 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -51,18 +51,106 @@ def _new_module(name): # Module-level locking ######################################################## -# A dict mapping module names to weakrefs of _ModuleLock instances -# Dictionary protected by the global import lock +# For a list that can have a weakref to it. +class _List(list): + pass + + +# Copied from weakref.py with some simplifications and modifications unique to +# bootstrapping importlib. Many methods were simply deleting for simplicity, so if they +# are needed in the future they may work if simply copied back in. +class _WeakValueDictionary: + + def __init__(self): + self_weakref = _weakref.ref(self) + + # Inlined to avoid issues with inheriting from _weakref.ref before _weakref is + # set by _setup(). Since there's only one instance of this class, this is + # not expensive. + class KeyedRef(_weakref.ref): + + __slots__ = "key", + + def __new__(type, ob, key): + self = super().__new__(type, ob, type.remove) + self.key = key + return self + + def __init__(self, ob, key): + super().__init__(ob, self.remove) + + @staticmethod + def remove(wr): + nonlocal self_weakref + + self = self_weakref() + if self is not None: + if self._iterating: + self._pending_removals.append(wr.key) + else: + _weakref._remove_dead_weakref(self.data, wr.key) + + self._KeyedRef = KeyedRef + self.clear() + + def clear(self): + self._pending_removals = [] + self._iterating = set() + self.data = {} + + def _commit_removals(self): + pop = self._pending_removals.pop + d = self.data + while True: + try: + key = pop() + except IndexError: + return + _weakref._remove_dead_weakref(d, key) + + def get(self, key, default=None): + if self._pending_removals: + self._commit_removals() + try: + wr = self.data[key] + except KeyError: + return default + else: + if (o := wr()) is None: + return default + else: + return o + + def setdefault(self, key, default=None): + try: + o = self.data[key]() + except KeyError: + o = None + if o is None: + if self._pending_removals: + self._commit_removals() + self.data[key] = self._KeyedRef(default, key) + return default + else: + return o + + +# A dict mapping module names to weakrefs of _ModuleLock instances. +# Dictionary protected by the global import lock. _module_locks = {} -# A dict mapping thread IDs to lists of _ModuleLock instances. This maps a -# thread to the module locks it is blocking on acquiring. The values are -# lists because a single thread could perform a re-entrant import and be "in -# the process" of blocking on locks for more than one module. A thread can -# be "in the process" because a thread cannot actually block on acquiring -# more than one lock but it can have set up bookkeeping that reflects that -# it intends to block on acquiring more than one lock. -_blocking_on = {} +# A dict mapping thread IDs to weakref'ed lists of _ModuleLock instances. +# This maps a thread to the module locks it is blocking on acquiring. The +# values are lists because a single thread could perform a re-entrant import +# and be "in the process" of blocking on locks for more than one module. A +# thread can be "in the process" because a thread cannot actually block on +# acquiring more than one lock but it can have set up bookkeeping that reflects +# that it intends to block on acquiring more than one lock. +# +# The dictionary uses a WeakValueDictionary to avoid keeping unnecessary +# lists around, regardless of GC runs. This way there's no memory leak if +# the list is no longer needed (GH-106176). +_blocking_on = None class _BlockingOnManager: @@ -79,7 +167,7 @@ def __enter__(self): # re-entrant (i.e., a single thread may take it more than once) so it # wouldn't help us be correct in the face of re-entrancy either. - self.blocked_on = _blocking_on.setdefault(self.thread_id, []) + self.blocked_on = _blocking_on.setdefault(self.thread_id, _List()) self.blocked_on.append(self.lock) def __exit__(self, *args, **kwargs): @@ -736,10 +824,16 @@ def _module_repr_from_spec(spec): """Return the repr to use for the module.""" name = '?' if spec.name is None else spec.name if spec.origin is None: - if spec.loader is None: + loader = spec.loader + if loader is None: return f'' + elif ( + _bootstrap_external is not None + and isinstance(loader, _bootstrap_external.NamespaceLoader) + ): + return f'' else: - return f'' + return f'' else: if spec.has_location: return f'' @@ -1409,7 +1503,7 @@ def _setup(sys_module, _imp_module): modules, those two modules must be explicitly passed in. """ - global _imp, sys + global _imp, sys, _blocking_on _imp = _imp_module sys = sys_module @@ -1437,6 +1531,9 @@ def _setup(sys_module, _imp_module): builtin_module = sys.modules[builtin_name] setattr(self_module, builtin_name, builtin_module) + # Instantiation requires _weakref to have been set. + _blocking_on = _WeakValueDictionary() + def _install(sys_module, _imp_module): """Install importers for builtin and frozen modules""" diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 3ca5a1aea28af7..4e96a916324824 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -444,8 +444,25 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) # Python 3.12b1 3529 (Inline list/dict/set comprehensions) # Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) - -# Python 3.13 will start with 3550 +# Python 3.12b1 3531 (Add PEP 695 changes) +# Python 3.13a1 3550 (Plugin optimizer support) +# Python 3.13a1 3551 (Compact superinstructions) +# Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) +# Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) +# Python 3.13a1 3554 (more efficient bytecodes for f-strings) +# Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) +# Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) +# Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) +# Python 3.13a1 3558 (Reorder the stack items for CALL) +# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) +# Python 3.13a1 3560 (Add RESUME_CHECK instruction) +# Python 3.13a1 3561 (Add cache entry to branch instructions) +# Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) +# Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) +# Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) +# Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) + +# Python 3.14 will start with 3600 # Please don't copy-paste the same pre-release tag for new entries above!!! # You should always use the *upcoming* tag. For example, if 3.12a6 came out @@ -460,7 +477,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3530).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3565).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 82e0ce1b281c54..5c09666b6a40d9 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -12,7 +12,6 @@ import functools import itertools import posixpath -import contextlib import collections import inspect diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index 34e3a9950cc557..e6b60c18caa052 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -6,31 +6,12 @@ Package, ) -from ._legacy import ( - contents, - open_binary, - read_binary, - open_text, - read_text, - is_resource, - path, - Resource, -) - from .abc import ResourceReader __all__ = [ 'Package', - 'Resource', 'ResourceReader', 'as_file', - 'contents', 'files', - 'is_resource', - 'open_binary', - 'open_text', - 'path', - 'read_binary', - 'read_text', ] diff --git a/Lib/importlib/resources/_legacy.py b/Lib/importlib/resources/_legacy.py deleted file mode 100644 index b1ea8105dad6e2..00000000000000 --- a/Lib/importlib/resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index b1d9271f8e47ca..3ad71d31c2f438 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -116,24 +116,40 @@ def find_spec(name, package=None): # is imported by runpy, which means we want to avoid any unnecessary # dependencies. Thus we use a class. -class allowing_all_extensions: - """A context manager that lets users skip the compatibility check. +class _incompatible_extension_module_restrictions: + """A context manager that can temporarily skip the compatibility check. + + NOTE: This function is meant to accommodate an unusual case; one + which is likely to eventually go away. There's is a pretty good + chance this is not what you were looking for. + + WARNING: Using this function to disable the check can lead to + unexpected behavior and even crashes. It should only be used during + extension module development. + + If "disable_check" is True then the compatibility check will not + happen while the context manager is active. Otherwise the check + *will* happen. Normally, extensions that do not support multiple interpreters may not be imported in a subinterpreter. That implies modules - that do not implement multi-phase init. + that do not implement multi-phase init or that explicitly of out. - Likewise for modules import in a subinterpeter with its own GIL + Likewise for modules import in a subinterpreter with its own GIL when the extension does not support a per-interpreter GIL. This implies the module does not have a Py_mod_multiple_interpreters slot set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. In both cases, this context manager may be used to temporarily disable the check for compatible extension modules. + + You can get the same effect as this function by implementing the + basic interface of multi-phase init (PEP 489) and lying about + support for mulitple interpreters (or per-interpreter GIL). """ - def __init__(self, disable_check=True): - self.disable_check = disable_check + def __init__(self, *, disable_check): + self.disable_check = bool(disable_check) def __enter__(self): self.old = _imp._override_multi_interp_extensions_check(self.override) diff --git a/Lib/inspect.py b/Lib/inspect.py index a64e85e4fd67a4..f0b72662a9a0b2 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -881,29 +881,28 @@ def cleandoc(doc): Any whitespace that can be uniformly removed from the second line onwards is removed.""" - try: - lines = doc.expandtabs().split('\n') - except UnicodeError: - return None - else: - # Find minimum indentation of any non-blank lines after first line. - margin = sys.maxsize - for line in lines[1:]: - content = len(line.lstrip()) - if content: - indent = len(line) - content - margin = min(margin, indent) - # Remove indentation. - if lines: - lines[0] = lines[0].lstrip() - if margin < sys.maxsize: - for i in range(1, len(lines)): lines[i] = lines[i][margin:] - # Remove any trailing or leading blank lines. - while lines and not lines[-1]: - lines.pop() - while lines and not lines[0]: - lines.pop(0) - return '\n'.join(lines) + lines = doc.expandtabs().split('\n') + + # Find minimum indentation of any non-blank lines after first line. + margin = sys.maxsize + for line in lines[1:]: + content = len(line.lstrip(' ')) + if content: + indent = len(line) - content + margin = min(margin, indent) + # Remove indentation. + if lines: + lines[0] = lines[0].lstrip(' ') + if margin < sys.maxsize: + for i in range(1, len(lines)): + lines[i] = lines[i][margin:] + # Remove any trailing or leading blank lines. + while lines and not lines[-1]: + lines.pop() + while lines and not lines[0]: + lines.pop(0) + return '\n'.join(lines) + def getfile(object): """Work out which source or compiled file an object was defined in.""" @@ -1035,9 +1034,13 @@ class ClassFoundException(Exception): class _ClassFinder(ast.NodeVisitor): - def __init__(self, qualname): + def __init__(self, cls, tree, lines, qualname): self.stack = [] + self.cls = cls + self.tree = tree + self.lines = lines self.qualname = qualname + self.lineno_found = [] def visit_FunctionDef(self, node): self.stack.append(node.name) @@ -1058,11 +1061,49 @@ def visit_ClassDef(self, node): line_number = node.lineno # decrement by one since lines starts with indexing by zero - line_number -= 1 - raise ClassFoundException(line_number) + self.lineno_found.append((line_number - 1, node.end_lineno)) self.generic_visit(node) self.stack.pop() + def get_lineno(self): + self.visit(self.tree) + lineno_found_number = len(self.lineno_found) + if lineno_found_number == 0: + raise OSError('could not find class definition') + elif lineno_found_number == 1: + return self.lineno_found[0][0] + else: + # We have multiple candidates for the class definition. + # Now we have to guess. + + # First, let's see if there are any method definitions + for member in self.cls.__dict__.values(): + if (isinstance(member, types.FunctionType) and + member.__module__ == self.cls.__module__): + for lineno, end_lineno in self.lineno_found: + if lineno <= member.__code__.co_firstlineno <= end_lineno: + return lineno + + class_strings = [(''.join(self.lines[lineno: end_lineno]), lineno) + for lineno, end_lineno in self.lineno_found] + + # Maybe the class has a docstring and it's unique? + if self.cls.__doc__: + ret = None + for candidate, lineno in class_strings: + if self.cls.__doc__.strip() in candidate: + if ret is None: + ret = lineno + else: + break + else: + if ret is not None: + return ret + + # We are out of ideas, just return the last one found, which is + # slightly better than previous ones + return self.lineno_found[-1][0] + def findsource(object): """Return the entire source file and starting line number for an object. @@ -1099,14 +1140,8 @@ def findsource(object): qualname = object.__qualname__ source = ''.join(lines) tree = ast.parse(source) - class_finder = _ClassFinder(qualname) - try: - class_finder.visit(tree) - except ClassFoundException as e: - line_number = e.args[0] - return lines, line_number - else: - raise OSError('could not find class definition') + class_finder = _ClassFinder(object, tree, lines, qualname) + return lines, class_finder.get_lineno() if ismethod(object): object = object.__func__ @@ -1242,6 +1277,14 @@ def getblock(lines): blockfinder.tokeneater(*_token) except (EndOfBlock, IndentationError): pass + except SyntaxError as e: + if "unmatched" not in e.msg: + raise e from None + _, *_token_info = _token + try: + blockfinder.tokeneater(tokenize.NEWLINE, *_token_info) + except (EndOfBlock, IndentationError): + pass return lines[:blockfinder.last] def getsourcelines(object): @@ -1835,8 +1878,10 @@ def getattr_static(obj, attr, default=_sentinel): klass_result = _check_class(klass, attr) if instance_result is not _sentinel and klass_result is not _sentinel: - if (_check_class(type(klass_result), '__get__') is not _sentinel and - _check_class(type(klass_result), '__set__') is not _sentinel): + if _check_class(type(klass_result), "__get__") is not _sentinel and ( + _check_class(type(klass_result), "__set__") is not _sentinel + or _check_class(type(klass_result), "__delete__") is not _sentinel + ): return klass_result if instance_result is not _sentinel: @@ -2185,7 +2230,7 @@ def _signature_strip_non_python_syntax(signature): if string == ',': current_parameter += 1 - if (type == ERRORTOKEN) and (string == '$'): + if (type == OP) and (string == '$'): assert self_parameter is None self_parameter = current_parameter continue @@ -2193,7 +2238,7 @@ def _signature_strip_non_python_syntax(signature): add(string) if (string == ','): add(' ') - clean_signature = ''.join(text) + clean_signature = ''.join(text).strip().replace("\n", "") return clean_signature, self_parameter @@ -2571,17 +2616,18 @@ def _signature_from_callable(obj, *, factory_method = None new = _signature_get_user_defined_method(obj, '__new__') init = _signature_get_user_defined_method(obj, '__init__') - # Now we check if the 'obj' class has an own '__new__' method - if '__new__' in obj.__dict__: - factory_method = new - # or an own '__init__' method - elif '__init__' in obj.__dict__: - factory_method = init - # If not, we take inherited '__new__' or '__init__', if present - elif new is not None: - factory_method = new - elif init is not None: - factory_method = init + + # Go through the MRO and see if any class has user-defined + # pure Python __new__ or __init__ method + for base in obj.__mro__: + # Now we check if the 'obj' class has an own '__new__' method + if new is not None and '__new__' in base.__dict__: + factory_method = new + break + # or an own '__init__' method + elif init is not None and '__init__' in base.__dict__: + factory_method = init + break if factory_method is not None: sig = _get_signature_of(factory_method) @@ -2824,6 +2870,8 @@ def __str__(self): return formatted + __replace__ = replace + def __repr__(self): return '<{} "{}">'.format(self.__class__.__name__, self) @@ -3084,6 +3132,8 @@ def replace(self, *, parameters=_void, return_annotation=_void): return type(self)(parameters, return_annotation=return_annotation) + __replace__ = replace + def _hash_basis(self): params = tuple(param for param in self.parameters.values() if param.kind != _KEYWORD_ONLY) @@ -3266,6 +3316,16 @@ def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, self) def __str__(self): + return self.format() + + def format(self, *, max_width=None): + """Create a string representation of the Signature object. + + If *max_width* integer is passed, + signature will try to fit into the *max_width*. + If signature is longer than *max_width*, + all parameters will be on separate lines. + """ result = [] render_pos_only_separator = False render_kw_only_separator = True @@ -3303,6 +3363,8 @@ def __str__(self): result.append('/') rendered = '({})'.format(', '.join(result)) + if max_width is not None and len(rendered) > max_width: + rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: anno = formatannotation(self.return_annotation) diff --git a/Lib/io.py b/Lib/io.py index 50ce97436ac1d1..f0e2fa15d5abcf 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -45,7 +45,8 @@ "FileIO", "BytesIO", "StringIO", "BufferedIOBase", "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper", - "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END"] + "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END", + "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder"] import _io diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index af1d5c4800cce8..e398cc138308d9 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1389,6 +1389,16 @@ def is_link_local(self): """ return self in self._constants._linklocal_network + @property + def ipv6_mapped(self): + """Return the IPv4-mapped IPv6 address. + + Returns: + The IPv4-mapped IPv6 address per RFC 4291. + + """ + return IPv6Address(f'::ffff:{self}') + class IPv4Interface(IPv4Address): @@ -1923,8 +1933,40 @@ def __init__(self, address): self._ip = self._ip_int_from_string(addr_str) + def _explode_shorthand_ip_string(self): + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + long_form = super()._explode_shorthand_ip_string() + else: + prefix_len = 30 + raw_exploded_str = super()._explode_shorthand_ip_string() + long_form = "%s%s" % (raw_exploded_str[:prefix_len], str(ipv4_mapped)) + return long_form + + def _ipv4_mapped_ipv6_to_str(self): + """Return convenient text representation of IPv4-mapped IPv6 address + + See RFC 4291 2.5.5.2, 2.2 p.3 for details. + + Returns: + A string, 'x:x:x:x:x:x:d.d.d.d', where the 'x's are the hexadecimal values of + the six high-order 16-bit pieces of the address, and the 'd's are + the decimal values of the four low-order 8-bit pieces of the + address (standard IPv4 representation) as defined in RFC 4291 2.2 p.3. + + """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + raise AddressValueError("Can not apply to non-IPv4-mapped IPv6 address %s" % str(self)) + high_order_bits = self._ip >> 32 + return "%s:%s" % (self._string_from_ip_int(high_order_bits), str(ipv4_mapped)) + def __str__(self): - ip_str = super().__str__() + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + ip_str = super().__str__() + else: + ip_str = self._ipv4_mapped_ipv6_to_str() return ip_str + '%' + self._scope_id if self._scope_id else ip_str def __hash__(self): @@ -1938,6 +1980,9 @@ def __eq__(self, other): return False return self._scope_id == getattr(other, '_scope_id', None) + def __reduce__(self): + return (self.__class__, (str(self),)) + @property def scope_id(self): """Identifier of a particular zone of the address's scope. diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 256e76a0a67f8f..ed2c74771ea87d 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -97,7 +97,7 @@ """ __version__ = '2.0.9' __all__ = [ - 'dump', 'dumps', 'load', 'loads', 'AttrDict', + 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', ] @@ -357,53 +357,3 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, if parse_constant is not None: kw['parse_constant'] = parse_constant return cls(**kw).decode(s) - -class AttrDict(dict): - """Dict like object that supports attribute style dotted access. - - This class is intended for use with the *object_hook* in json.loads(): - - >>> from json import loads, AttrDict - >>> json_string = '{"mercury": 88, "venus": 225, "earth": 365, "mars": 687}' - >>> orbital_period = loads(json_string, object_hook=AttrDict) - >>> orbital_period['earth'] # Dict style lookup - 365 - >>> orbital_period.earth # Attribute style lookup - 365 - >>> orbital_period.keys() # All dict methods are present - dict_keys(['mercury', 'venus', 'earth', 'mars']) - - Attribute style access only works for keys that are valid attribute names. - In contrast, dictionary style access works for all keys. - For example, ``d.two words`` contains a space and is not syntactically - valid Python, so ``d["two words"]`` should be used instead. - - If a key has the same name as dictionary method, then a dictionary - lookup finds the key and an attribute lookup finds the method: - - >>> d = AttrDict(items=50) - >>> d['items'] # Lookup the key - 50 - >>> d.items() # Call the method - dict_items([('items', 50)]) - - """ - __slots__ = () - - def __getattr__(self, attr): - try: - return self[attr] - except KeyError: - raise AttributeError(attr) from None - - def __setattr__(self, attr, value): - self[attr] = value - - def __delattr__(self, attr): - try: - del self[attr] - except KeyError: - raise AttributeError(attr) from None - - def __dir__(self): - return list(self) + dir(type(self)) diff --git a/Lib/keyword.py b/Lib/keyword.py index cc2b46b7229d53..e22c837835e740 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -56,7 +56,8 @@ softkwlist = [ '_', 'case', - 'match' + 'match', + 'type' ] iskeyword = frozenset(kwlist).__contains__ diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt deleted file mode 100644 index fa7b15061d941c..00000000000000 --- a/Lib/lib2to3/Grammar.txt +++ /dev/null @@ -1,196 +0,0 @@ -# Grammar for 2to3. This grammar supports Python 2.x and 3.x. - -# NOTE WELL: You should also follow all the steps listed at -# https://devguide.python.org/grammar/ - -# Start symbols for the grammar: -# file_input is a module or sequence of commands read from an input file; -# single_input is a single interactive statement; -# eval_input is the input for the eval() and input() functions. -# NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE -eval_input: testlist NEWLINE* ENDMARKER - -decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE -decorators: decorator+ -decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: ASYNC funcdef -funcdef: 'def' NAME parameters ['->' test] ':' suite -parameters: '(' [typedargslist] ')' - -# The following definition for typedarglist is equivalent to this set of rules: -# -# arguments = argument (',' argument)* -# argument = tfpdef ['=' test] -# kwargs = '**' tname [','] -# args = '*' [tname] -# kwonly_kwargs = (',' argument)* [',' [kwargs]] -# args_kwonly_kwargs = args kwonly_kwargs | kwargs -# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] -# typedargslist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs -# typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])|(typedargslist_no_posonly)" -# -# It needs to be fully expanded to allow our LL(1) parser to work on it. - -typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [ - ',' [((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])* - [',' ['**' tname [',']]] | '**' tname [',']) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])] - ] | ((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])* - [',' ['**' tname [',']]] | '**' tname [',']) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) - -tname: NAME [':' test] -tfpdef: tname | '(' tfplist ')' -tfplist: tfpdef (',' tfpdef)* [','] - -# The following definition for varargslist is equivalent to this set of rules: -# -# arguments = argument (',' argument )* -# argument = vfpdef ['=' test] -# kwargs = '**' vname [','] -# args = '*' [vname] -# kwonly_kwargs = (',' argument )* [',' [kwargs]] -# args_kwonly_kwargs = args kwonly_kwargs | kwargs -# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] -# vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs -# varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly) -# -# It needs to be fully expanded to allow our LL(1) parser to work on it. - -varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ - ((vfpdef ['=' test] ',')* ('*' [vname] (',' vname ['=' test])* - [',' ['**' vname [',']]] | '**' vname [',']) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - ]] | ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]]| '**' vname [',']) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - -vname: NAME -vfpdef: vname | '(' vfplist ')' -vfplist: vfpdef (',' vfpdef)* [','] - -stmt: simple_stmt | compound_stmt -simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | - import_stmt | global_stmt | exec_stmt | assert_stmt) -expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) -annassign: ':' test ['=' test] -testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] -augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | - '<<=' | '>>=' | '**=' | '//=') -# For normal and annotated assignments, additional restrictions enforced by the interpreter -print_stmt: 'print' ( [ test (',' test)* [','] ] | - '>>' test [ (',' test)+ [','] ] ) -del_stmt: 'del' exprlist -pass_stmt: 'pass' -flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt -break_stmt: 'break' -continue_stmt: 'continue' -return_stmt: 'return' [testlist_star_expr] -yield_stmt: yield_expr -raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] -import_stmt: import_name | import_from -import_name: 'import' dotted_as_names -import_from: ('from' ('.'* dotted_name | '.'+) - 'import' ('*' | '(' import_as_names ')' | import_as_names)) -import_as_name: NAME ['as' NAME] -dotted_as_name: dotted_name ['as' NAME] -import_as_names: import_as_name (',' import_as_name)* [','] -dotted_as_names: dotted_as_name (',' dotted_as_name)* -dotted_name: NAME ('.' NAME)* -global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* -exec_stmt: 'exec' expr ['in' test [',' test]] -assert_stmt: 'assert' test [',' test] - -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) -if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] -while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] -for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] -try_stmt: ('try' ':' suite - ((except_clause ':' suite)+ - ['else' ':' suite] - ['finally' ':' suite] | - 'finally' ':' suite)) -with_stmt: 'with' with_item (',' with_item)* ':' suite -with_item: test ['as' expr] -with_var: 'as' expr -# NB compile.c makes sure that the default except clause is last -except_clause: 'except' [test [(',' | 'as') test]] -suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT - -# Backward compatibility cruft to support: -# [ x for x in lambda: True, lambda: False if x() ] -# even while also allowing: -# lambda x: 5 if x else 2 -# (But not a mix of the two) -testlist_safe: old_test [(',' old_test)+ [',']] -old_test: or_test | old_lambdef -old_lambdef: 'lambda' [varargslist] ':' old_test - -namedexpr_test: test [':=' test] -test: or_test ['if' or_test 'else' test] | lambdef -or_test: and_test ('or' and_test)* -and_test: not_test ('and' not_test)* -not_test: 'not' not_test | comparison -comparison: expr (comp_op expr)* -comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -star_expr: '*' expr -expr: xor_expr ('|' xor_expr)* -xor_expr: and_expr ('^' and_expr)* -and_expr: shift_expr ('&' shift_expr)* -shift_expr: arith_expr (('<<'|'>>') arith_expr)* -arith_expr: term (('+'|'-') term)* -term: factor (('*'|'@'|'/'|'%'|'//') factor)* -factor: ('+'|'-'|'~') factor | power -power: [AWAIT] atom trailer* ['**' factor] -atom: ('(' [yield_expr|testlist_gexp] ')' | - '[' [listmaker] ']' | - '{' [dictsetmaker] '}' | - '`' testlist1 '`' | - NAME | NUMBER | STRING+ | '.' '.' '.') -listmaker: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) -testlist_gexp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) -lambdef: 'lambda' [varargslist] ':' test -trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -subscriptlist: subscript (',' subscript)* [','] -subscript: test | [test] ':' [test] [sliceop] -sliceop: ':' [test] -exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] -testlist: test (',' test)* [','] -dictsetmaker: ( ((test ':' test | '**' expr) - (comp_for | (',' (test ':' test | '**' expr))* [','])) | - ((test | star_expr) - (comp_for | (',' (test | star_expr))* [','])) ) - -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite - -arglist: argument (',' argument)* [','] - -# "test '=' test" is really "keyword '=' test", but we have no such token. -# These need to be in a single rule to avoid grammar that is ambiguous -# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, -# we explicitly match '*' here, too, to give it proper precedence. -# Illegal combinations and orderings are blocked in ast.c: -# multiple (test comp_for) arguments are blocked; keyword unpackings -# that precede iterable unpackings are blocked; etc. -argument: ( test [comp_for] | - test ':=' test | - test '=' test | - '**' test | - '*' test ) - -comp_iter: comp_for | comp_if -comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [comp_iter] -comp_if: 'if' old_test [comp_iter] - -testlist1: test (',' test)* - -# not used in grammar, but may appear in "node" passed from Parser to Compiler -encoding_decl: NAME - -yield_expr: 'yield' [yield_arg] -yield_arg: 'from' test | testlist_star_expr diff --git a/Lib/lib2to3/PatternGrammar.txt b/Lib/lib2to3/PatternGrammar.txt deleted file mode 100644 index 36bf8148273bd7..00000000000000 --- a/Lib/lib2to3/PatternGrammar.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# A grammar to describe tree matching patterns. -# Not shown here: -# - 'TOKEN' stands for any token (leaf node) -# - 'any' stands for any node (leaf or interior) -# With 'any' we can still specify the sub-structure. - -# The start symbol is 'Matcher'. - -Matcher: Alternatives ENDMARKER - -Alternatives: Alternative ('|' Alternative)* - -Alternative: (Unit | NegatedUnit)+ - -Unit: [NAME '='] ( STRING [Repeater] - | NAME [Details] [Repeater] - | '(' Alternatives ')' [Repeater] - | '[' Alternatives ']' - ) - -NegatedUnit: 'not' (STRING | NAME [Details] | '(' Alternatives ')') - -Repeater: '*' | '+' | '{' NUMBER [',' NUMBER] '}' - -Details: '<' Alternatives '>' diff --git a/Lib/lib2to3/__init__.py b/Lib/lib2to3/__init__.py deleted file mode 100644 index 177405c8090d3e..00000000000000 --- a/Lib/lib2to3/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -import warnings - - -warnings.warn( - "lib2to3 package is deprecated and may not be able to parse Python 3.10+", - DeprecationWarning, - stacklevel=2, -) diff --git a/Lib/lib2to3/__main__.py b/Lib/lib2to3/__main__.py deleted file mode 100644 index 80688baf27abfc..00000000000000 --- a/Lib/lib2to3/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys -from .main import main - -sys.exit(main("lib2to3.fixes")) diff --git a/Lib/lib2to3/btm_matcher.py b/Lib/lib2to3/btm_matcher.py deleted file mode 100644 index 3b78868038bda0..00000000000000 --- a/Lib/lib2to3/btm_matcher.py +++ /dev/null @@ -1,163 +0,0 @@ -"""A bottom-up tree matching algorithm implementation meant to speed -up 2to3's matching process. After the tree patterns are reduced to -their rarest linear path, a linear Aho-Corasick automaton is -created. The linear automaton traverses the linear paths from the -leaves to the root of the AST and returns a set of nodes for further -matching. This reduces significantly the number of candidate nodes.""" - -__author__ = "George Boutsioukis " - -import logging -import itertools -from collections import defaultdict - -from . import pytree -from .btm_utils import reduce_tree - -class BMNode(object): - """Class for a node of the Aho-Corasick automaton used in matching""" - count = itertools.count() - def __init__(self): - self.transition_table = {} - self.fixers = [] - self.id = next(BMNode.count) - self.content = '' - -class BottomMatcher(object): - """The main matcher class. After instantiating the patterns should - be added using the add_fixer method""" - - def __init__(self): - self.match = set() - self.root = BMNode() - self.nodes = [self.root] - self.fixers = [] - self.logger = logging.getLogger("RefactoringTool") - - def add_fixer(self, fixer): - """Reduces a fixer's pattern tree to a linear path and adds it - to the matcher(a common Aho-Corasick automaton). The fixer is - appended on the matching states and called when they are - reached""" - self.fixers.append(fixer) - tree = reduce_tree(fixer.pattern_tree) - linear = tree.get_linear_subpattern() - match_nodes = self.add(linear, start=self.root) - for match_node in match_nodes: - match_node.fixers.append(fixer) - - def add(self, pattern, start): - "Recursively adds a linear pattern to the AC automaton" - #print("adding pattern", pattern, "to", start) - if not pattern: - #print("empty pattern") - return [start] - if isinstance(pattern[0], tuple): - #alternatives - #print("alternatives") - match_nodes = [] - for alternative in pattern[0]: - #add all alternatives, and add the rest of the pattern - #to each end node - end_nodes = self.add(alternative, start=start) - for end in end_nodes: - match_nodes.extend(self.add(pattern[1:], end)) - return match_nodes - else: - #single token - #not last - if pattern[0] not in start.transition_table: - #transition did not exist, create new - next_node = BMNode() - start.transition_table[pattern[0]] = next_node - else: - #transition exists already, follow - next_node = start.transition_table[pattern[0]] - - if pattern[1:]: - end_nodes = self.add(pattern[1:], start=next_node) - else: - end_nodes = [next_node] - return end_nodes - - def run(self, leaves): - """The main interface with the bottom matcher. The tree is - traversed from the bottom using the constructed - automaton. Nodes are only checked once as the tree is - retraversed. When the automaton fails, we give it one more - shot(in case the above tree matches as a whole with the - rejected leaf), then we break for the next leaf. There is the - special case of multiple arguments(see code comments) where we - recheck the nodes - - Args: - The leaves of the AST tree to be matched - - Returns: - A dictionary of node matches with fixers as the keys - """ - current_ac_node = self.root - results = defaultdict(list) - for leaf in leaves: - current_ast_node = leaf - while current_ast_node: - current_ast_node.was_checked = True - for child in current_ast_node.children: - # multiple statements, recheck - if isinstance(child, pytree.Leaf) and child.value == ";": - current_ast_node.was_checked = False - break - if current_ast_node.type == 1: - #name - node_token = current_ast_node.value - else: - node_token = current_ast_node.type - - if node_token in current_ac_node.transition_table: - #token matches - current_ac_node = current_ac_node.transition_table[node_token] - for fixer in current_ac_node.fixers: - results[fixer].append(current_ast_node) - else: - #matching failed, reset automaton - current_ac_node = self.root - if (current_ast_node.parent is not None - and current_ast_node.parent.was_checked): - #the rest of the tree upwards has been checked, next leaf - break - - #recheck the rejected node once from the root - if node_token in current_ac_node.transition_table: - #token matches - current_ac_node = current_ac_node.transition_table[node_token] - for fixer in current_ac_node.fixers: - results[fixer].append(current_ast_node) - - current_ast_node = current_ast_node.parent - return results - - def print_ac(self): - "Prints a graphviz diagram of the BM automaton(for debugging)" - print("digraph g{") - def print_node(node): - for subnode_key in node.transition_table.keys(): - subnode = node.transition_table[subnode_key] - print("%d -> %d [label=%s] //%s" % - (node.id, subnode.id, type_repr(subnode_key), str(subnode.fixers))) - if subnode_key == 1: - print(subnode.content) - print_node(subnode) - print_node(self.root) - print("}") - -# taken from pytree.py for debugging; only used by print_ac -_type_reprs = {} -def type_repr(type_num): - global _type_reprs - if not _type_reprs: - from .pygram import python_symbols - # printing tokens is possible but not as useful - # from .pgen2 import token // token.__dict__.items(): - for name, val in python_symbols.__dict__.items(): - if type(val) == int: _type_reprs[val] = name - return _type_reprs.setdefault(type_num, type_num) diff --git a/Lib/lib2to3/btm_utils.py b/Lib/lib2to3/btm_utils.py deleted file mode 100644 index b61afdba693071..00000000000000 --- a/Lib/lib2to3/btm_utils.py +++ /dev/null @@ -1,280 +0,0 @@ -"Utility functions used by the btm_matcher module" - -from . import pytree -from .pgen2 import grammar, token -from .pygram import pattern_symbols, python_symbols - -syms = pattern_symbols -pysyms = python_symbols -tokens = grammar.opmap -token_labels = token - -TYPE_ANY = -1 -TYPE_ALTERNATIVES = -2 -TYPE_GROUP = -3 - -class MinNode(object): - """This class serves as an intermediate representation of the - pattern tree during the conversion to sets of leaf-to-root - subpatterns""" - - def __init__(self, type=None, name=None): - self.type = type - self.name = name - self.children = [] - self.leaf = False - self.parent = None - self.alternatives = [] - self.group = [] - - def __repr__(self): - return str(self.type) + ' ' + str(self.name) - - def leaf_to_root(self): - """Internal method. Returns a characteristic path of the - pattern tree. This method must be run for all leaves until the - linear subpatterns are merged into a single""" - node = self - subp = [] - while node: - if node.type == TYPE_ALTERNATIVES: - node.alternatives.append(subp) - if len(node.alternatives) == len(node.children): - #last alternative - subp = [tuple(node.alternatives)] - node.alternatives = [] - node = node.parent - continue - else: - node = node.parent - subp = None - break - - if node.type == TYPE_GROUP: - node.group.append(subp) - #probably should check the number of leaves - if len(node.group) == len(node.children): - subp = get_characteristic_subpattern(node.group) - node.group = [] - node = node.parent - continue - else: - node = node.parent - subp = None - break - - if node.type == token_labels.NAME and node.name: - #in case of type=name, use the name instead - subp.append(node.name) - else: - subp.append(node.type) - - node = node.parent - return subp - - def get_linear_subpattern(self): - """Drives the leaf_to_root method. The reason that - leaf_to_root must be run multiple times is because we need to - reject 'group' matches; for example the alternative form - (a | b c) creates a group [b c] that needs to be matched. Since - matching multiple linear patterns overcomes the automaton's - capabilities, leaf_to_root merges each group into a single - choice based on 'characteristic'ity, - - i.e. (a|b c) -> (a|b) if b more characteristic than c - - Returns: The most 'characteristic'(as defined by - get_characteristic_subpattern) path for the compiled pattern - tree. - """ - - for l in self.leaves(): - subp = l.leaf_to_root() - if subp: - return subp - - def leaves(self): - "Generator that returns the leaves of the tree" - for child in self.children: - yield from child.leaves() - if not self.children: - yield self - -def reduce_tree(node, parent=None): - """ - Internal function. Reduces a compiled pattern tree to an - intermediate representation suitable for feeding the - automaton. This also trims off any optional pattern elements(like - [a], a*). - """ - - new_node = None - #switch on the node type - if node.type == syms.Matcher: - #skip - node = node.children[0] - - if node.type == syms.Alternatives : - #2 cases - if len(node.children) <= 2: - #just a single 'Alternative', skip this node - new_node = reduce_tree(node.children[0], parent) - else: - #real alternatives - new_node = MinNode(type=TYPE_ALTERNATIVES) - #skip odd children('|' tokens) - for child in node.children: - if node.children.index(child)%2: - continue - reduced = reduce_tree(child, new_node) - if reduced is not None: - new_node.children.append(reduced) - elif node.type == syms.Alternative: - if len(node.children) > 1: - - new_node = MinNode(type=TYPE_GROUP) - for child in node.children: - reduced = reduce_tree(child, new_node) - if reduced: - new_node.children.append(reduced) - if not new_node.children: - # delete the group if all of the children were reduced to None - new_node = None - - else: - new_node = reduce_tree(node.children[0], parent) - - elif node.type == syms.Unit: - if (isinstance(node.children[0], pytree.Leaf) and - node.children[0].value == '('): - #skip parentheses - return reduce_tree(node.children[1], parent) - if ((isinstance(node.children[0], pytree.Leaf) and - node.children[0].value == '[') - or - (len(node.children)>1 and - hasattr(node.children[1], "value") and - node.children[1].value == '[')): - #skip whole unit if its optional - return None - - leaf = True - details_node = None - alternatives_node = None - has_repeater = False - repeater_node = None - has_variable_name = False - - for child in node.children: - if child.type == syms.Details: - leaf = False - details_node = child - elif child.type == syms.Repeater: - has_repeater = True - repeater_node = child - elif child.type == syms.Alternatives: - alternatives_node = child - if hasattr(child, 'value') and child.value == '=': # variable name - has_variable_name = True - - #skip variable name - if has_variable_name: - #skip variable name, '=' - name_leaf = node.children[2] - if hasattr(name_leaf, 'value') and name_leaf.value == '(': - # skip parenthesis - name_leaf = node.children[3] - else: - name_leaf = node.children[0] - - #set node type - if name_leaf.type == token_labels.NAME: - #(python) non-name or wildcard - if name_leaf.value == 'any': - new_node = MinNode(type=TYPE_ANY) - else: - if hasattr(token_labels, name_leaf.value): - new_node = MinNode(type=getattr(token_labels, name_leaf.value)) - else: - new_node = MinNode(type=getattr(pysyms, name_leaf.value)) - - elif name_leaf.type == token_labels.STRING: - #(python) name or character; remove the apostrophes from - #the string value - name = name_leaf.value.strip("'") - if name in tokens: - new_node = MinNode(type=tokens[name]) - else: - new_node = MinNode(type=token_labels.NAME, name=name) - elif name_leaf.type == syms.Alternatives: - new_node = reduce_tree(alternatives_node, parent) - - #handle repeaters - if has_repeater: - if repeater_node.children[0].value == '*': - #reduce to None - new_node = None - elif repeater_node.children[0].value == '+': - #reduce to a single occurrence i.e. do nothing - pass - else: - #TODO: handle {min, max} repeaters - raise NotImplementedError - - #add children - if details_node and new_node is not None: - for child in details_node.children[1:-1]: - #skip '<', '>' markers - reduced = reduce_tree(child, new_node) - if reduced is not None: - new_node.children.append(reduced) - if new_node: - new_node.parent = parent - return new_node - - -def get_characteristic_subpattern(subpatterns): - """Picks the most characteristic from a list of linear patterns - Current order used is: - names > common_names > common_chars - """ - if not isinstance(subpatterns, list): - return subpatterns - if len(subpatterns)==1: - return subpatterns[0] - - # first pick out the ones containing variable names - subpatterns_with_names = [] - subpatterns_with_common_names = [] - common_names = ['in', 'for', 'if' , 'not', 'None'] - subpatterns_with_common_chars = [] - common_chars = "[]().,:" - for subpattern in subpatterns: - if any(rec_test(subpattern, lambda x: type(x) is str)): - if any(rec_test(subpattern, - lambda x: isinstance(x, str) and x in common_chars)): - subpatterns_with_common_chars.append(subpattern) - elif any(rec_test(subpattern, - lambda x: isinstance(x, str) and x in common_names)): - subpatterns_with_common_names.append(subpattern) - - else: - subpatterns_with_names.append(subpattern) - - if subpatterns_with_names: - subpatterns = subpatterns_with_names - elif subpatterns_with_common_names: - subpatterns = subpatterns_with_common_names - elif subpatterns_with_common_chars: - subpatterns = subpatterns_with_common_chars - # of the remaining subpatterns pick out the longest one - return max(subpatterns, key=len) - -def rec_test(sequence, test_func): - """Tests test_func on all items of sequence and items of included - sub-iterables""" - for x in sequence: - if isinstance(x, (list, tuple)): - yield from rec_test(x, test_func) - else: - yield test_func(x) diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py deleted file mode 100644 index df581a4deab9e9..00000000000000 --- a/Lib/lib2to3/fixer_base.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Base class for fixers (optional, but recommended).""" - -# Python imports -import itertools - -# Local imports -from .patcomp import PatternCompiler -from . import pygram -from .fixer_util import does_tree_import - -class BaseFix(object): - - """Optional base class for fixers. - - The subclass name must be FixFooBar where FooBar is the result of - removing underscores and capitalizing the words of the fix name. - For example, the class name for a fixer named 'has_key' should be - FixHasKey. - """ - - PATTERN = None # Most subclasses should override with a string literal - pattern = None # Compiled pattern, set by compile_pattern() - pattern_tree = None # Tree representation of the pattern - options = None # Options object passed to initializer - filename = None # The filename (set by set_filename) - numbers = itertools.count(1) # For new_name() - used_names = set() # A set of all used NAMEs - order = "post" # Does the fixer prefer pre- or post-order traversal - explicit = False # Is this ignored by refactor.py -f all? - run_order = 5 # Fixers will be sorted by run order before execution - # Lower numbers will be run first. - _accept_type = None # [Advanced and not public] This tells RefactoringTool - # which node type to accept when there's not a pattern. - - keep_line_order = False # For the bottom matcher: match with the - # original line order - BM_compatible = False # Compatibility with the bottom matching - # module; every fixer should set this - # manually - - # Shortcut for access to Python grammar symbols - syms = pygram.python_symbols - - def __init__(self, options, log): - """Initializer. Subclass may override. - - Args: - options: a dict containing the options passed to RefactoringTool - that could be used to customize the fixer through the command line. - log: a list to append warnings and other messages to. - """ - self.options = options - self.log = log - self.compile_pattern() - - def compile_pattern(self): - """Compiles self.PATTERN into self.pattern. - - Subclass may override if it doesn't want to use - self.{pattern,PATTERN} in .match(). - """ - if self.PATTERN is not None: - PC = PatternCompiler() - self.pattern, self.pattern_tree = PC.compile_pattern(self.PATTERN, - with_tree=True) - - def set_filename(self, filename): - """Set the filename. - - The main refactoring tool should call this. - """ - self.filename = filename - - def match(self, node): - """Returns match for a given parse tree node. - - Should return a true or false object (not necessarily a bool). - It may return a non-empty dict of matching sub-nodes as - returned by a matching pattern. - - Subclass may override. - """ - results = {"node": node} - return self.pattern.match(node, results) and results - - def transform(self, node, results): - """Returns the transformation for a given parse tree node. - - Args: - node: the root of the parse tree that matched the fixer. - results: a dict mapping symbolic names to part of the match. - - Returns: - None, or a node that is a modified copy of the - argument node. The node argument may also be modified in-place to - effect the same change. - - Subclass *must* override. - """ - raise NotImplementedError() - - def new_name(self, template="xxx_todo_changeme"): - """Return a string suitable for use as an identifier - - The new name is guaranteed not to conflict with other identifiers. - """ - name = template - while name in self.used_names: - name = template + str(next(self.numbers)) - self.used_names.add(name) - return name - - def log_message(self, message): - if self.first_log: - self.first_log = False - self.log.append("### In file %s ###" % self.filename) - self.log.append(message) - - def cannot_convert(self, node, reason=None): - """Warn the user that a given chunk of code is not valid Python 3, - but that it cannot be converted automatically. - - First argument is the top-level node for the code in question. - Optional second argument is why it can't be converted. - """ - lineno = node.get_lineno() - for_output = node.clone() - for_output.prefix = "" - msg = "Line %d: could not convert: %s" - self.log_message(msg % (lineno, for_output)) - if reason: - self.log_message(reason) - - def warning(self, node, reason): - """Used for warning the user about possible uncertainty in the - translation. - - First argument is the top-level node for the code in question. - Optional second argument is why it can't be converted. - """ - lineno = node.get_lineno() - self.log_message("Line %d: %s" % (lineno, reason)) - - def start_tree(self, tree, filename): - """Some fixers need to maintain tree-wide state. - This method is called once, at the start of tree fix-up. - - tree - the root node of the tree to be processed. - filename - the name of the file the tree came from. - """ - self.used_names = tree.used_names - self.set_filename(filename) - self.numbers = itertools.count(1) - self.first_log = True - - def finish_tree(self, tree, filename): - """Some fixers need to maintain tree-wide state. - This method is called once, at the conclusion of tree fix-up. - - tree - the root node of the tree to be processed. - filename - the name of the file the tree came from. - """ - pass - - -class ConditionalFix(BaseFix): - """ Base class for fixers which not execute if an import is found. """ - - # This is the name of the import which, if found, will cause the test to be skipped - skip_on = None - - def start_tree(self, *args): - super(ConditionalFix, self).start_tree(*args) - self._should_skip = None - - def should_skip(self, node): - if self._should_skip is not None: - return self._should_skip - pkg = self.skip_on.split(".") - name = pkg[-1] - pkg = ".".join(pkg[:-1]) - self._should_skip = does_tree_import(pkg, name, node) - return self._should_skip diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py deleted file mode 100644 index c2a3a47f503286..00000000000000 --- a/Lib/lib2to3/fixer_util.py +++ /dev/null @@ -1,453 +0,0 @@ -"""Utility functions, node construction macros, etc.""" -# Author: Collin Winter - -# Local imports -from .pgen2 import token -from .pytree import Leaf, Node -from .pygram import python_symbols as syms -from . import patcomp - - -########################################################### -### Common node-construction "macros" -########################################################### - -def KeywordArg(keyword, value): - return Node(syms.argument, - [keyword, Leaf(token.EQUAL, "="), value]) - -def LParen(): - return Leaf(token.LPAR, "(") - -def RParen(): - return Leaf(token.RPAR, ")") - -def Assign(target, source): - """Build an assignment statement""" - if not isinstance(target, list): - target = [target] - if not isinstance(source, list): - source.prefix = " " - source = [source] - - return Node(syms.atom, - target + [Leaf(token.EQUAL, "=", prefix=" ")] + source) - -def Name(name, prefix=None): - """Return a NAME leaf""" - return Leaf(token.NAME, name, prefix=prefix) - -def Attr(obj, attr): - """A node tuple for obj.attr""" - return [obj, Node(syms.trailer, [Dot(), attr])] - -def Comma(): - """A comma leaf""" - return Leaf(token.COMMA, ",") - -def Dot(): - """A period (.) leaf""" - return Leaf(token.DOT, ".") - -def ArgList(args, lparen=LParen(), rparen=RParen()): - """A parenthesised argument list, used by Call()""" - node = Node(syms.trailer, [lparen.clone(), rparen.clone()]) - if args: - node.insert_child(1, Node(syms.arglist, args)) - return node - -def Call(func_name, args=None, prefix=None): - """A function call""" - node = Node(syms.power, [func_name, ArgList(args)]) - if prefix is not None: - node.prefix = prefix - return node - -def Newline(): - """A newline literal""" - return Leaf(token.NEWLINE, "\n") - -def BlankLine(): - """A blank line""" - return Leaf(token.NEWLINE, "") - -def Number(n, prefix=None): - return Leaf(token.NUMBER, n, prefix=prefix) - -def Subscript(index_node): - """A numeric or string subscript""" - return Node(syms.trailer, [Leaf(token.LBRACE, "["), - index_node, - Leaf(token.RBRACE, "]")]) - -def String(string, prefix=None): - """A string leaf""" - return Leaf(token.STRING, string, prefix=prefix) - -def ListComp(xp, fp, it, test=None): - """A list comprehension of the form [xp for fp in it if test]. - - If test is None, the "if test" part is omitted. - """ - xp.prefix = "" - fp.prefix = " " - it.prefix = " " - for_leaf = Leaf(token.NAME, "for") - for_leaf.prefix = " " - in_leaf = Leaf(token.NAME, "in") - in_leaf.prefix = " " - inner_args = [for_leaf, fp, in_leaf, it] - if test: - test.prefix = " " - if_leaf = Leaf(token.NAME, "if") - if_leaf.prefix = " " - inner_args.append(Node(syms.comp_if, [if_leaf, test])) - inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)]) - return Node(syms.atom, - [Leaf(token.LBRACE, "["), - inner, - Leaf(token.RBRACE, "]")]) - -def FromImport(package_name, name_leafs): - """ Return an import statement in the form: - from package import name_leafs""" - # XXX: May not handle dotted imports properly (eg, package_name='foo.bar') - #assert package_name == '.' or '.' not in package_name, "FromImport has "\ - # "not been tested with dotted package names -- use at your own "\ - # "peril!" - - for leaf in name_leafs: - # Pull the leaves out of their old tree - leaf.remove() - - children = [Leaf(token.NAME, "from"), - Leaf(token.NAME, package_name, prefix=" "), - Leaf(token.NAME, "import", prefix=" "), - Node(syms.import_as_names, name_leafs)] - imp = Node(syms.import_from, children) - return imp - -def ImportAndCall(node, results, names): - """Returns an import statement and calls a method - of the module: - - import module - module.name()""" - obj = results["obj"].clone() - if obj.type == syms.arglist: - newarglist = obj.clone() - else: - newarglist = Node(syms.arglist, [obj.clone()]) - after = results["after"] - if after: - after = [n.clone() for n in after] - new = Node(syms.power, - Attr(Name(names[0]), Name(names[1])) + - [Node(syms.trailer, - [results["lpar"].clone(), - newarglist, - results["rpar"].clone()])] + after) - new.prefix = node.prefix - return new - - -########################################################### -### Determine whether a node represents a given literal -########################################################### - -def is_tuple(node): - """Does the node represent a tuple literal?""" - if isinstance(node, Node) and node.children == [LParen(), RParen()]: - return True - return (isinstance(node, Node) - and len(node.children) == 3 - and isinstance(node.children[0], Leaf) - and isinstance(node.children[1], Node) - and isinstance(node.children[2], Leaf) - and node.children[0].value == "(" - and node.children[2].value == ")") - -def is_list(node): - """Does the node represent a list literal?""" - return (isinstance(node, Node) - and len(node.children) > 1 - and isinstance(node.children[0], Leaf) - and isinstance(node.children[-1], Leaf) - and node.children[0].value == "[" - and node.children[-1].value == "]") - - -########################################################### -### Misc -########################################################### - -def parenthesize(node): - return Node(syms.atom, [LParen(), node, RParen()]) - - -consuming_calls = {"sorted", "list", "set", "any", "all", "tuple", "sum", - "min", "max", "enumerate"} - -def attr_chain(obj, attr): - """Follow an attribute chain. - - If you have a chain of objects where a.foo -> b, b.foo-> c, etc, - use this to iterate over all objects in the chain. Iteration is - terminated by getattr(x, attr) is None. - - Args: - obj: the starting object - attr: the name of the chaining attribute - - Yields: - Each successive object in the chain. - """ - next = getattr(obj, attr) - while next: - yield next - next = getattr(next, attr) - -p0 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - """ -p1 = """ -power< - ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' | - 'any' | 'all' | 'enumerate' | (any* trailer< '.' 'join' >) ) - trailer< '(' node=any ')' > - any* -> -""" -p2 = """ -power< - ( 'sorted' | 'enumerate' ) - trailer< '(' arglist ')' > - any* -> -""" -pats_built = False -def in_special_context(node): - """ Returns true if node is in an environment where all that is required - of it is being iterable (ie, it doesn't matter if it returns a list - or an iterator). - See test_map_nochange in test_fixers.py for some examples and tests. - """ - global p0, p1, p2, pats_built - if not pats_built: - p0 = patcomp.compile_pattern(p0) - p1 = patcomp.compile_pattern(p1) - p2 = patcomp.compile_pattern(p2) - pats_built = True - patterns = [p0, p1, p2] - for pattern, parent in zip(patterns, attr_chain(node, "parent")): - results = {} - if pattern.match(parent, results) and results["node"] is node: - return True - return False - -def is_probably_builtin(node): - """ - Check that something isn't an attribute or function name etc. - """ - prev = node.prev_sibling - if prev is not None and prev.type == token.DOT: - # Attribute lookup. - return False - parent = node.parent - if parent.type in (syms.funcdef, syms.classdef): - return False - if parent.type == syms.expr_stmt and parent.children[0] is node: - # Assignment. - return False - if parent.type == syms.parameters or \ - (parent.type == syms.typedargslist and ( - (prev is not None and prev.type == token.COMMA) or - parent.children[0] is node - )): - # The name of an argument. - return False - return True - -def find_indentation(node): - """Find the indentation of *node*.""" - while node is not None: - if node.type == syms.suite and len(node.children) > 2: - indent = node.children[1] - if indent.type == token.INDENT: - return indent.value - node = node.parent - return "" - -########################################################### -### The following functions are to find bindings in a suite -########################################################### - -def make_suite(node): - if node.type == syms.suite: - return node - node = node.clone() - parent, node.parent = node.parent, None - suite = Node(syms.suite, [node]) - suite.parent = parent - return suite - -def find_root(node): - """Find the top level namespace.""" - # Scamper up to the top level namespace - while node.type != syms.file_input: - node = node.parent - if not node: - raise ValueError("root found before file_input node was found.") - return node - -def does_tree_import(package, name, node): - """ Returns true if name is imported from package at the - top level of the tree which node belongs to. - To cover the case of an import like 'import foo', use - None for the package and 'foo' for the name. """ - binding = find_binding(name, find_root(node), package) - return bool(binding) - -def is_import(node): - """Returns true if the node is an import statement.""" - return node.type in (syms.import_name, syms.import_from) - -def touch_import(package, name, node): - """ Works like `does_tree_import` but adds an import statement - if it was not imported. """ - def is_import_stmt(node): - return (node.type == syms.simple_stmt and node.children and - is_import(node.children[0])) - - root = find_root(node) - - if does_tree_import(package, name, root): - return - - # figure out where to insert the new import. First try to find - # the first import and then skip to the last one. - insert_pos = offset = 0 - for idx, node in enumerate(root.children): - if not is_import_stmt(node): - continue - for offset, node2 in enumerate(root.children[idx:]): - if not is_import_stmt(node2): - break - insert_pos = idx + offset - break - - # if there are no imports where we can insert, find the docstring. - # if that also fails, we stick to the beginning of the file - if insert_pos == 0: - for idx, node in enumerate(root.children): - if (node.type == syms.simple_stmt and node.children and - node.children[0].type == token.STRING): - insert_pos = idx + 1 - break - - if package is None: - import_ = Node(syms.import_name, [ - Leaf(token.NAME, "import"), - Leaf(token.NAME, name, prefix=" ") - ]) - else: - import_ = FromImport(package, [Leaf(token.NAME, name, prefix=" ")]) - - children = [import_, Newline()] - root.insert_child(insert_pos, Node(syms.simple_stmt, children)) - - -_def_syms = {syms.classdef, syms.funcdef} -def find_binding(name, node, package=None): - """ Returns the node which binds variable name, otherwise None. - If optional argument package is supplied, only imports will - be returned. - See test cases for examples.""" - for child in node.children: - ret = None - if child.type == syms.for_stmt: - if _find(name, child.children[1]): - return child - n = find_binding(name, make_suite(child.children[-1]), package) - if n: ret = n - elif child.type in (syms.if_stmt, syms.while_stmt): - n = find_binding(name, make_suite(child.children[-1]), package) - if n: ret = n - elif child.type == syms.try_stmt: - n = find_binding(name, make_suite(child.children[2]), package) - if n: - ret = n - else: - for i, kid in enumerate(child.children[3:]): - if kid.type == token.COLON and kid.value == ":": - # i+3 is the colon, i+4 is the suite - n = find_binding(name, make_suite(child.children[i+4]), package) - if n: ret = n - elif child.type in _def_syms and child.children[1].value == name: - ret = child - elif _is_import_binding(child, name, package): - ret = child - elif child.type == syms.simple_stmt: - ret = find_binding(name, child, package) - elif child.type == syms.expr_stmt: - if _find(name, child.children[0]): - ret = child - - if ret: - if not package: - return ret - if is_import(ret): - return ret - return None - -_block_syms = {syms.funcdef, syms.classdef, syms.trailer} -def _find(name, node): - nodes = [node] - while nodes: - node = nodes.pop() - if node.type > 256 and node.type not in _block_syms: - nodes.extend(node.children) - elif node.type == token.NAME and node.value == name: - return node - return None - -def _is_import_binding(node, name, package=None): - """ Will return node if node will import name, or node - will import * from package. None is returned otherwise. - See test cases for examples. """ - - if node.type == syms.import_name and not package: - imp = node.children[1] - if imp.type == syms.dotted_as_names: - for child in imp.children: - if child.type == syms.dotted_as_name: - if child.children[2].value == name: - return node - elif child.type == token.NAME and child.value == name: - return node - elif imp.type == syms.dotted_as_name: - last = imp.children[-1] - if last.type == token.NAME and last.value == name: - return node - elif imp.type == token.NAME and imp.value == name: - return node - elif node.type == syms.import_from: - # str(...) is used to make life easier here, because - # from a.b import parses to ['import', ['a', '.', 'b'], ...] - if package and str(node.children[1]).strip() != package: - return None - n = node.children[3] - if package and _find("as", n): - # See test_from_import_as for explanation - return None - elif n.type == syms.import_as_names and _find(name, n): - return node - elif n.type == syms.import_as_name: - child = n.children[2] - if child.type == token.NAME and child.value == name: - return node - elif n.type == token.NAME and n.value == name: - return node - elif package and n.type == token.STAR: - return node - return None diff --git a/Lib/lib2to3/fixes/__init__.py b/Lib/lib2to3/fixes/__init__.py deleted file mode 100644 index b93054b3ecf3a5..00000000000000 --- a/Lib/lib2to3/fixes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Dummy file to make this directory a package. diff --git a/Lib/lib2to3/fixes/fix_apply.py b/Lib/lib2to3/fixes/fix_apply.py deleted file mode 100644 index 6408582c426477..00000000000000 --- a/Lib/lib2to3/fixes/fix_apply.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for apply(). - -This converts apply(func, v, k) into (func)(*v, **k).""" - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Call, Comma, parenthesize - -class FixApply(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'apply' - trailer< - '(' - arglist< - (not argument - ')' - > - > - """ - - def transform(self, node, results): - syms = self.syms - assert results - func = results["func"] - args = results["args"] - kwds = results.get("kwds") - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - if args: - if (args.type == self.syms.argument and - args.children[0].value in {'**', '*'}): - return # Make no change. - if kwds and (kwds.type == self.syms.argument and - kwds.children[0].value == '**'): - return # Make no change. - prefix = node.prefix - func = func.clone() - if (func.type not in (token.NAME, syms.atom) and - (func.type != syms.power or - func.children[-2].type == token.DOUBLESTAR)): - # Need to parenthesize - func = parenthesize(func) - func.prefix = "" - args = args.clone() - args.prefix = "" - if kwds is not None: - kwds = kwds.clone() - kwds.prefix = "" - l_newargs = [pytree.Leaf(token.STAR, "*"), args] - if kwds is not None: - l_newargs.extend([Comma(), - pytree.Leaf(token.DOUBLESTAR, "**"), - kwds]) - l_newargs[-2].prefix = " " # that's the ** token - # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t) - # can be translated into f(x, y, *t) instead of f(*(x, y) + t) - #new = pytree.Node(syms.power, (func, ArgList(l_newargs))) - return Call(func, l_newargs, prefix=prefix) diff --git a/Lib/lib2to3/fixes/fix_asserts.py b/Lib/lib2to3/fixes/fix_asserts.py deleted file mode 100644 index 5bcec885f52cbf..00000000000000 --- a/Lib/lib2to3/fixes/fix_asserts.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Fixer that replaces deprecated unittest method names.""" - -# Author: Ezio Melotti - -from ..fixer_base import BaseFix -from ..fixer_util import Name - -NAMES = dict( - assert_="assertTrue", - assertEquals="assertEqual", - assertNotEquals="assertNotEqual", - assertAlmostEquals="assertAlmostEqual", - assertNotAlmostEquals="assertNotAlmostEqual", - assertRegexpMatches="assertRegex", - assertRaisesRegexp="assertRaisesRegex", - failUnlessEqual="assertEqual", - failIfEqual="assertNotEqual", - failUnlessAlmostEqual="assertAlmostEqual", - failIfAlmostEqual="assertNotAlmostEqual", - failUnless="assertTrue", - failUnlessRaises="assertRaises", - failIf="assertFalse", -) - - -class FixAsserts(BaseFix): - - PATTERN = """ - power< any+ trailer< '.' meth=(%s)> any* > - """ % '|'.join(map(repr, NAMES)) - - def transform(self, node, results): - name = results["meth"][0] - name.replace(Name(NAMES[str(name)], prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_basestring.py b/Lib/lib2to3/fixes/fix_basestring.py deleted file mode 100644 index 5fe69a0f03b1b8..00000000000000 --- a/Lib/lib2to3/fixes/fix_basestring.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Fixer for basestring -> str.""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixBasestring(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = "'basestring'" - - def transform(self, node, results): - return Name("str", prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_buffer.py b/Lib/lib2to3/fixes/fix_buffer.py deleted file mode 100644 index f9a1958ad3b93e..00000000000000 --- a/Lib/lib2to3/fixes/fix_buffer.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes buffer(...) into memoryview(...).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixBuffer(fixer_base.BaseFix): - BM_compatible = True - - explicit = True # The user must ask for this fixer - - PATTERN = """ - power< name='buffer' trailer< '(' [any] ')' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("memoryview", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_dict.py b/Lib/lib2to3/fixes/fix_dict.py deleted file mode 100644 index d3655c9f1b2d9b..00000000000000 --- a/Lib/lib2to3/fixes/fix_dict.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for dict methods. - -d.keys() -> list(d.keys()) -d.items() -> list(d.items()) -d.values() -> list(d.values()) - -d.iterkeys() -> iter(d.keys()) -d.iteritems() -> iter(d.items()) -d.itervalues() -> iter(d.values()) - -d.viewkeys() -> d.keys() -d.viewitems() -> d.items() -d.viewvalues() -> d.values() - -Except in certain very specific contexts: the iter() can be dropped -when the context is list(), sorted(), iter() or for...in; the list() -can be dropped when the context is list() or sorted() (but not iter() -or for...in!). Special contexts that apply to both: list(), sorted(), tuple() -set(), any(), all(), sum(). - -Note: iter(d.keys()) could be written as iter(d) but since the -original d.iterkeys() was also redundant we don't fix this. And there -are (rare) contexts where it makes a difference (e.g. when passing it -as an argument to a function that introspects the argument). -""" - -# Local imports -from .. import pytree -from .. import patcomp -from .. import fixer_base -from ..fixer_util import Name, Call, Dot -from .. import fixer_util - - -iter_exempt = fixer_util.consuming_calls | {"iter"} - - -class FixDict(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< head=any+ - trailer< '.' method=('keys'|'items'|'values'| - 'iterkeys'|'iteritems'|'itervalues'| - 'viewkeys'|'viewitems'|'viewvalues') > - parens=trailer< '(' ')' > - tail=any* - > - """ - - def transform(self, node, results): - head = results["head"] - method = results["method"][0] # Extract node for method name - tail = results["tail"] - syms = self.syms - method_name = method.value - isiter = method_name.startswith("iter") - isview = method_name.startswith("view") - if isiter or isview: - method_name = method_name[4:] - assert method_name in ("keys", "items", "values"), repr(method) - head = [n.clone() for n in head] - tail = [n.clone() for n in tail] - special = not tail and self.in_special_context(node, isiter) - args = head + [pytree.Node(syms.trailer, - [Dot(), - Name(method_name, - prefix=method.prefix)]), - results["parens"].clone()] - new = pytree.Node(syms.power, args) - if not (special or isview): - new.prefix = "" - new = Call(Name("iter" if isiter else "list"), [new]) - if tail: - new = pytree.Node(syms.power, [new] + tail) - new.prefix = node.prefix - return new - - P1 = "power< func=NAME trailer< '(' node=any ')' > any* >" - p1 = patcomp.compile_pattern(P1) - - P2 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - """ - p2 = patcomp.compile_pattern(P2) - - def in_special_context(self, node, isiter): - if node.parent is None: - return False - results = {} - if (node.parent.parent is not None and - self.p1.match(node.parent.parent, results) and - results["node"] is node): - if isiter: - # iter(d.iterkeys()) -> iter(d.keys()), etc. - return results["func"].value in iter_exempt - else: - # list(d.keys()) -> list(d.keys()), etc. - return results["func"].value in fixer_util.consuming_calls - if not isiter: - return False - # for ... in d.iterkeys() -> for ... in d.keys(), etc. - return self.p2.match(node.parent, results) and results["node"] is node diff --git a/Lib/lib2to3/fixes/fix_except.py b/Lib/lib2to3/fixes/fix_except.py deleted file mode 100644 index 49bd3d5ab7d6cc..00000000000000 --- a/Lib/lib2to3/fixes/fix_except.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Fixer for except statements with named exceptions. - -The following cases will be converted: - -- "except E, T:" where T is a name: - - except E as T: - -- "except E, T:" where T is not a name, tuple or list: - - except E as t: - T = t - - This is done because the target of an "except" clause must be a - name. - -- "except E, T:" where T is a tuple or list literal: - - except E as t: - T = t.args -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, syms - -def find_excepts(nodes): - for i, n in enumerate(nodes): - if n.type == syms.except_clause: - if n.children[0].value == 'except': - yield (n, nodes[i+2]) - -class FixExcept(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - try_stmt< 'try' ':' (simple_stmt | suite) - cleanup=(except_clause ':' (simple_stmt | suite))+ - tail=(['except' ':' (simple_stmt | suite)] - ['else' ':' (simple_stmt | suite)] - ['finally' ':' (simple_stmt | suite)]) > - """ - - def transform(self, node, results): - syms = self.syms - - tail = [n.clone() for n in results["tail"]] - - try_cleanup = [ch.clone() for ch in results["cleanup"]] - for except_clause, e_suite in find_excepts(try_cleanup): - if len(except_clause.children) == 4: - (E, comma, N) = except_clause.children[1:4] - comma.replace(Name("as", prefix=" ")) - - if N.type != token.NAME: - # Generate a new N for the except clause - new_N = Name(self.new_name(), prefix=" ") - target = N.clone() - target.prefix = "" - N.replace(new_N) - new_N = new_N.clone() - - # Insert "old_N = new_N" as the first statement in - # the except body. This loop skips leading whitespace - # and indents - #TODO(cwinter) suite-cleanup - suite_stmts = e_suite.children - for i, stmt in enumerate(suite_stmts): - if isinstance(stmt, pytree.Node): - break - - # The assignment is different if old_N is a tuple or list - # In that case, the assignment is old_N = new_N.args - if is_tuple(N) or is_list(N): - assign = Assign(target, Attr(new_N, Name('args'))) - else: - assign = Assign(target, new_N) - - #TODO(cwinter) stopgap until children becomes a smart list - for child in reversed(suite_stmts[:i]): - e_suite.insert_child(0, child) - e_suite.insert_child(i, assign) - elif N.prefix == "": - # No space after a comma is legal; no space after "as", - # not so much. - N.prefix = " " - - #TODO(cwinter) fix this when children becomes a smart list - children = [c.clone() for c in node.children[:3]] + try_cleanup + tail - return pytree.Node(node.type, children) diff --git a/Lib/lib2to3/fixes/fix_exec.py b/Lib/lib2to3/fixes/fix_exec.py deleted file mode 100644 index ab921ee80cdf36..00000000000000 --- a/Lib/lib2to3/fixes/fix_exec.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for exec. - -This converts usages of the exec statement into calls to a built-in -exec() function. - -exec code in ns1, ns2 -> exec(code, ns1, ns2) -""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Comma, Name, Call - - -class FixExec(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - exec_stmt< 'exec' a=any 'in' b=any [',' c=any] > - | - exec_stmt< 'exec' (not atom<'(' [any] ')'>) a=any > - """ - - def transform(self, node, results): - assert results - syms = self.syms - a = results["a"] - b = results.get("b") - c = results.get("c") - args = [a.clone()] - args[0].prefix = "" - if b is not None: - args.extend([Comma(), b.clone()]) - if c is not None: - args.extend([Comma(), c.clone()]) - - return Call(Name("exec"), args, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_execfile.py b/Lib/lib2to3/fixes/fix_execfile.py deleted file mode 100644 index b6c786fd4e8b6a..00000000000000 --- a/Lib/lib2to3/fixes/fix_execfile.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for execfile. - -This converts usages of the execfile function into calls to the built-in -exec() function. -""" - -from .. import fixer_base -from ..fixer_util import (Comma, Name, Call, LParen, RParen, Dot, Node, - ArgList, String, syms) - - -class FixExecfile(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'execfile' trailer< '(' arglist< filename=any [',' globals=any [',' locals=any ] ] > ')' > > - | - power< 'execfile' trailer< '(' filename=any ')' > > - """ - - def transform(self, node, results): - assert results - filename = results["filename"] - globals = results.get("globals") - locals = results.get("locals") - - # Copy over the prefix from the right parentheses end of the execfile - # call. - execfile_paren = node.children[-1].children[-1].clone() - # Construct open().read(). - open_args = ArgList([filename.clone(), Comma(), String('"rb"', ' ')], - rparen=execfile_paren) - open_call = Node(syms.power, [Name("open"), open_args]) - read = [Node(syms.trailer, [Dot(), Name('read')]), - Node(syms.trailer, [LParen(), RParen()])] - open_expr = [open_call] + read - # Wrap the open call in a compile call. This is so the filename will be - # preserved in the execed code. - filename_arg = filename.clone() - filename_arg.prefix = " " - exec_str = String("'exec'", " ") - compile_args = open_expr + [Comma(), filename_arg, Comma(), exec_str] - compile_call = Call(Name("compile"), compile_args, "") - # Finally, replace the execfile call with an exec call. - args = [compile_call] - if globals is not None: - args.extend([Comma(), globals.clone()]) - if locals is not None: - args.extend([Comma(), locals.clone()]) - return Call(Name("exec"), args, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_exitfunc.py b/Lib/lib2to3/fixes/fix_exitfunc.py deleted file mode 100644 index 2e47887afead36..00000000000000 --- a/Lib/lib2to3/fixes/fix_exitfunc.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Convert use of sys.exitfunc to use the atexit module. -""" - -# Author: Benjamin Peterson - -from lib2to3 import pytree, fixer_base -from lib2to3.fixer_util import Name, Attr, Call, Comma, Newline, syms - - -class FixExitfunc(fixer_base.BaseFix): - keep_line_order = True - BM_compatible = True - - PATTERN = """ - ( - sys_import=import_name<'import' - ('sys' - | - dotted_as_names< (any ',')* 'sys' (',' any)* > - ) - > - | - expr_stmt< - power< 'sys' trailer< '.' 'exitfunc' > > - '=' func=any > - ) - """ - - def __init__(self, *args): - super(FixExitfunc, self).__init__(*args) - - def start_tree(self, tree, filename): - super(FixExitfunc, self).start_tree(tree, filename) - self.sys_import = None - - def transform(self, node, results): - # First, find the sys import. We'll just hope it's global scope. - if "sys_import" in results: - if self.sys_import is None: - self.sys_import = results["sys_import"] - return - - func = results["func"].clone() - func.prefix = "" - register = pytree.Node(syms.power, - Attr(Name("atexit"), Name("register")) - ) - call = Call(register, [func], node.prefix) - node.replace(call) - - if self.sys_import is None: - # That's interesting. - self.warning(node, "Can't find sys import; Please add an atexit " - "import at the top of your file.") - return - - # Now add an atexit import after the sys import. - names = self.sys_import.children[1] - if names.type == syms.dotted_as_names: - names.append_child(Comma()) - names.append_child(Name("atexit", " ")) - else: - containing_stmt = self.sys_import.parent - position = containing_stmt.children.index(self.sys_import) - stmt_container = containing_stmt.parent - new_import = pytree.Node(syms.import_name, - [Name("import"), Name("atexit", " ")] - ) - new = pytree.Node(syms.simple_stmt, [new_import]) - containing_stmt.insert_child(position + 1, Newline()) - containing_stmt.insert_child(position + 2, new) diff --git a/Lib/lib2to3/fixes/fix_filter.py b/Lib/lib2to3/fixes/fix_filter.py deleted file mode 100644 index 38e9078f11ac88..00000000000000 --- a/Lib/lib2to3/fixes/fix_filter.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes filter(F, X) into list(filter(F, X)). - -We avoid the transformation if the filter() call is directly contained -in iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or -for V in <>:. - -NOTE: This is still not correct if the original code was depending on -filter(F, X) to return a string if X is a string and a tuple if X is a -tuple. That would require type inference, which we don't do. Let -Python 2.6 figure it out. -""" - -# Local imports -from .. import fixer_base -from ..pytree import Node -from ..pygram import python_symbols as syms -from ..fixer_util import Name, ArgList, ListComp, in_special_context, parenthesize - - -class FixFilter(fixer_base.ConditionalFix): - BM_compatible = True - - PATTERN = """ - filter_lambda=power< - 'filter' - trailer< - '(' - arglist< - lambdef< 'lambda' - (fp=NAME | vfpdef< '(' fp=NAME ')'> ) ':' xp=any - > - ',' - it=any - > - ')' - > - [extra_trailers=trailer*] - > - | - power< - 'filter' - trailer< '(' arglist< none='None' ',' seq=any > ')' > - [extra_trailers=trailer*] - > - | - power< - 'filter' - args=trailer< '(' [any] ')' > - [extra_trailers=trailer*] - > - """ - - skip_on = "future_builtins.filter" - - def transform(self, node, results): - if self.should_skip(node): - return - - trailers = [] - if 'extra_trailers' in results: - for t in results['extra_trailers']: - trailers.append(t.clone()) - - if "filter_lambda" in results: - xp = results.get("xp").clone() - if xp.type == syms.test: - xp.prefix = "" - xp = parenthesize(xp) - - new = ListComp(results.get("fp").clone(), - results.get("fp").clone(), - results.get("it").clone(), xp) - new = Node(syms.power, [new] + trailers, prefix="") - - elif "none" in results: - new = ListComp(Name("_f"), - Name("_f"), - results["seq"].clone(), - Name("_f")) - new = Node(syms.power, [new] + trailers, prefix="") - - else: - if in_special_context(node): - return None - - args = results['args'].clone() - new = Node(syms.power, [Name("filter"), args], prefix="") - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = "" - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_funcattrs.py b/Lib/lib2to3/fixes/fix_funcattrs.py deleted file mode 100644 index 67f3e18e061bdb..00000000000000 --- a/Lib/lib2to3/fixes/fix_funcattrs.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Fix function attribute names (f.func_x -> f.__x__).""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixFuncattrs(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< any+ trailer< '.' attr=('func_closure' | 'func_doc' | 'func_globals' - | 'func_name' | 'func_defaults' | 'func_code' - | 'func_dict') > any* > - """ - - def transform(self, node, results): - attr = results["attr"][0] - attr.replace(Name(("__%s__" % attr.value[5:]), - prefix=attr.prefix)) diff --git a/Lib/lib2to3/fixes/fix_future.py b/Lib/lib2to3/fixes/fix_future.py deleted file mode 100644 index fbcb86af079133..00000000000000 --- a/Lib/lib2to3/fixes/fix_future.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Remove __future__ imports - -from __future__ import foo is replaced with an empty line. -""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import BlankLine - -class FixFuture(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """import_from< 'from' module_name="__future__" 'import' any >""" - - # This should be run last -- some things check for the import - run_order = 10 - - def transform(self, node, results): - new = BlankLine() - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_getcwdu.py b/Lib/lib2to3/fixes/fix_getcwdu.py deleted file mode 100644 index 087eaedcb26f9c..00000000000000 --- a/Lib/lib2to3/fixes/fix_getcwdu.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Fixer that changes os.getcwdu() to os.getcwd(). -""" -# Author: Victor Stinner - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixGetcwdu(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'os' trailer< dot='.' name='getcwdu' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("getcwd", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_has_key.py b/Lib/lib2to3/fixes/fix_has_key.py deleted file mode 100644 index 439708c9923404..00000000000000 --- a/Lib/lib2to3/fixes/fix_has_key.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for has_key(). - -Calls to .has_key() methods are expressed in terms of the 'in' -operator: - - d.has_key(k) -> k in d - -CAVEATS: -1) While the primary target of this fixer is dict.has_key(), the - fixer will change any has_key() method call, regardless of its - class. - -2) Cases like this will not be converted: - - m = d.has_key - if m(k): - ... - - Only *calls* to has_key() are converted. While it is possible to - convert the above to something like - - m = d.__contains__ - if m(k): - ... - - this is currently not done. -""" - -# Local imports -from .. import pytree -from .. import fixer_base -from ..fixer_util import Name, parenthesize - - -class FixHasKey(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - anchor=power< - before=any+ - trailer< '.' 'has_key' > - trailer< - '(' - ( not(arglist | argument) arg=any ','> - ) - ')' - > - after=any* - > - | - negation=not_test< - 'not' - anchor=power< - before=any+ - trailer< '.' 'has_key' > - trailer< - '(' - ( not(arglist | argument) arg=any ','> - ) - ')' - > - > - > - """ - - def transform(self, node, results): - assert results - syms = self.syms - if (node.parent.type == syms.not_test and - self.pattern.match(node.parent)): - # Don't transform a node matching the first alternative of the - # pattern when its parent matches the second alternative - return None - negation = results.get("negation") - anchor = results["anchor"] - prefix = node.prefix - before = [n.clone() for n in results["before"]] - arg = results["arg"].clone() - after = results.get("after") - if after: - after = [n.clone() for n in after] - if arg.type in (syms.comparison, syms.not_test, syms.and_test, - syms.or_test, syms.test, syms.lambdef, syms.argument): - arg = parenthesize(arg) - if len(before) == 1: - before = before[0] - else: - before = pytree.Node(syms.power, before) - before.prefix = " " - n_op = Name("in", prefix=" ") - if negation: - n_not = Name("not", prefix=" ") - n_op = pytree.Node(syms.comp_op, (n_not, n_op)) - new = pytree.Node(syms.comparison, (arg, n_op, before)) - if after: - new = parenthesize(new) - new = pytree.Node(syms.power, (new,) + tuple(after)) - if node.parent.type in (syms.comparison, syms.expr, syms.xor_expr, - syms.and_expr, syms.shift_expr, - syms.arith_expr, syms.term, - syms.factor, syms.power): - new = parenthesize(new) - new.prefix = prefix - return new diff --git a/Lib/lib2to3/fixes/fix_idioms.py b/Lib/lib2to3/fixes/fix_idioms.py deleted file mode 100644 index 6905913d7cb79d..00000000000000 --- a/Lib/lib2to3/fixes/fix_idioms.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Adjust some old Python 2 idioms to their modern counterparts. - -* Change some type comparisons to isinstance() calls: - type(x) == T -> isinstance(x, T) - type(x) is T -> isinstance(x, T) - type(x) != T -> not isinstance(x, T) - type(x) is not T -> not isinstance(x, T) - -* Change "while 1:" into "while True:". - -* Change both - - v = list(EXPR) - v.sort() - foo(v) - -and the more general - - v = EXPR - v.sort() - foo(v) - -into - - v = sorted(EXPR) - foo(v) -""" -# Author: Jacques Frechet, Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Comma, Name, Node, BlankLine, syms - -CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)" -TYPE = "power< 'type' trailer< '(' x=any ')' > >" - -class FixIdioms(fixer_base.BaseFix): - explicit = True # The user must ask for this fixer - - PATTERN = r""" - isinstance=comparison< %s %s T=any > - | - isinstance=comparison< T=any %s %s > - | - while_stmt< 'while' while='1' ':' any+ > - | - sorted=any< - any* - simple_stmt< - expr_stmt< id1=any '=' - power< list='list' trailer< '(' (not arglist) any ')' > > - > - '\n' - > - sort= - simple_stmt< - power< id2=any - trailer< '.' 'sort' > trailer< '(' ')' > - > - '\n' - > - next=any* - > - | - sorted=any< - any* - simple_stmt< expr_stmt< id1=any '=' expr=any > '\n' > - sort= - simple_stmt< - power< id2=any - trailer< '.' 'sort' > trailer< '(' ')' > - > - '\n' - > - next=any* - > - """ % (TYPE, CMP, CMP, TYPE) - - def match(self, node): - r = super(FixIdioms, self).match(node) - # If we've matched one of the sort/sorted subpatterns above, we - # want to reject matches where the initial assignment and the - # subsequent .sort() call involve different identifiers. - if r and "sorted" in r: - if r["id1"] == r["id2"]: - return r - return None - return r - - def transform(self, node, results): - if "isinstance" in results: - return self.transform_isinstance(node, results) - elif "while" in results: - return self.transform_while(node, results) - elif "sorted" in results: - return self.transform_sort(node, results) - else: - raise RuntimeError("Invalid match") - - def transform_isinstance(self, node, results): - x = results["x"].clone() # The thing inside of type() - T = results["T"].clone() # The type being compared against - x.prefix = "" - T.prefix = " " - test = Call(Name("isinstance"), [x, Comma(), T]) - if "n" in results: - test.prefix = " " - test = Node(syms.not_test, [Name("not"), test]) - test.prefix = node.prefix - return test - - def transform_while(self, node, results): - one = results["while"] - one.replace(Name("True", prefix=one.prefix)) - - def transform_sort(self, node, results): - sort_stmt = results["sort"] - next_stmt = results["next"] - list_call = results.get("list") - simple_expr = results.get("expr") - - if list_call: - list_call.replace(Name("sorted", prefix=list_call.prefix)) - elif simple_expr: - new = simple_expr.clone() - new.prefix = "" - simple_expr.replace(Call(Name("sorted"), [new], - prefix=simple_expr.prefix)) - else: - raise RuntimeError("should not have reached here") - sort_stmt.remove() - - btwn = sort_stmt.prefix - # Keep any prefix lines between the sort_stmt and the list_call and - # shove them right after the sorted() call. - if "\n" in btwn: - if next_stmt: - # The new prefix should be everything from the sort_stmt's - # prefix up to the last newline, then the old prefix after a new - # line. - prefix_lines = (btwn.rpartition("\n")[0], next_stmt[0].prefix) - next_stmt[0].prefix = "\n".join(prefix_lines) - else: - assert list_call.parent - assert list_call.next_sibling is None - # Put a blank line after list_call and set its prefix. - end_line = BlankLine() - list_call.parent.append_child(end_line) - assert list_call.next_sibling is end_line - # The new prefix should be everything up to the first new line - # of sort_stmt's prefix. - end_line.prefix = btwn.rpartition("\n")[0] diff --git a/Lib/lib2to3/fixes/fix_import.py b/Lib/lib2to3/fixes/fix_import.py deleted file mode 100644 index 734ca294699c36..00000000000000 --- a/Lib/lib2to3/fixes/fix_import.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Fixer for import statements. -If spam is being imported from the local directory, this import: - from spam import eggs -Becomes: - from .spam import eggs - -And this import: - import spam -Becomes: - from . import spam -""" - -# Local imports -from .. import fixer_base -from os.path import dirname, join, exists, sep -from ..fixer_util import FromImport, syms, token - - -def traverse_imports(names): - """ - Walks over all the names imported in a dotted_as_names node. - """ - pending = [names] - while pending: - node = pending.pop() - if node.type == token.NAME: - yield node.value - elif node.type == syms.dotted_name: - yield "".join([ch.value for ch in node.children]) - elif node.type == syms.dotted_as_name: - pending.append(node.children[0]) - elif node.type == syms.dotted_as_names: - pending.extend(node.children[::-2]) - else: - raise AssertionError("unknown node type") - - -class FixImport(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - import_from< 'from' imp=any 'import' ['('] any [')'] > - | - import_name< 'import' imp=any > - """ - - def start_tree(self, tree, name): - super(FixImport, self).start_tree(tree, name) - self.skip = "absolute_import" in tree.future_features - - def transform(self, node, results): - if self.skip: - return - imp = results['imp'] - - if node.type == syms.import_from: - # Some imps are top-level (eg: 'import ham') - # some are first level (eg: 'import ham.eggs') - # some are third level (eg: 'import ham.eggs as spam') - # Hence, the loop - while not hasattr(imp, 'value'): - imp = imp.children[0] - if self.probably_a_local_import(imp.value): - imp.value = "." + imp.value - imp.changed() - else: - have_local = False - have_absolute = False - for mod_name in traverse_imports(imp): - if self.probably_a_local_import(mod_name): - have_local = True - else: - have_absolute = True - if have_absolute: - if have_local: - # We won't handle both sibling and absolute imports in the - # same statement at the moment. - self.warning(node, "absolute and local imports together") - return - - new = FromImport(".", [imp]) - new.prefix = node.prefix - return new - - def probably_a_local_import(self, imp_name): - if imp_name.startswith("."): - # Relative imports are certainly not local imports. - return False - imp_name = imp_name.split(".", 1)[0] - base_path = dirname(self.filename) - base_path = join(base_path, imp_name) - # If there is no __init__.py next to the file its not in a package - # so can't be a relative import. - if not exists(join(dirname(base_path), "__init__.py")): - return False - for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]: - if exists(base_path + ext): - return True - return False diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py deleted file mode 100644 index aaf4f2f642efb5..00000000000000 --- a/Lib/lib2to3/fixes/fix_imports.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Fix incompatible imports and module references.""" -# Authors: Collin Winter, Nick Edds - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, attr_chain - -MAPPING = {'StringIO': 'io', - 'cStringIO': 'io', - 'cPickle': 'pickle', - '__builtin__' : 'builtins', - 'copy_reg': 'copyreg', - 'Queue': 'queue', - 'SocketServer': 'socketserver', - 'ConfigParser': 'configparser', - 'repr': 'reprlib', - 'FileDialog': 'tkinter.filedialog', - 'tkFileDialog': 'tkinter.filedialog', - 'SimpleDialog': 'tkinter.simpledialog', - 'tkSimpleDialog': 'tkinter.simpledialog', - 'tkColorChooser': 'tkinter.colorchooser', - 'tkCommonDialog': 'tkinter.commondialog', - 'Dialog': 'tkinter.dialog', - 'Tkdnd': 'tkinter.dnd', - 'tkFont': 'tkinter.font', - 'tkMessageBox': 'tkinter.messagebox', - 'ScrolledText': 'tkinter.scrolledtext', - 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', - 'ttk': 'tkinter.ttk', - 'Tkinter': 'tkinter', - 'markupbase': '_markupbase', - '_winreg': 'winreg', - 'thread': '_thread', - 'dummy_thread': '_dummy_thread', - # anydbm and whichdb are handled by fix_imports2 - 'dbhash': 'dbm.bsd', - 'dumbdbm': 'dbm.dumb', - 'dbm': 'dbm.ndbm', - 'gdbm': 'dbm.gnu', - 'xmlrpclib': 'xmlrpc.client', - 'DocXMLRPCServer': 'xmlrpc.server', - 'SimpleXMLRPCServer': 'xmlrpc.server', - 'httplib': 'http.client', - 'htmlentitydefs' : 'html.entities', - 'HTMLParser' : 'html.parser', - 'Cookie': 'http.cookies', - 'cookielib': 'http.cookiejar', - 'BaseHTTPServer': 'http.server', - 'SimpleHTTPServer': 'http.server', - 'CGIHTTPServer': 'http.server', - #'test.test_support': 'test.support', - 'commands': 'subprocess', - 'UserString' : 'collections', - 'UserList' : 'collections', - 'urlparse' : 'urllib.parse', - 'robotparser' : 'urllib.robotparser', -} - - -def alternates(members): - return "(" + "|".join(map(repr, members)) + ")" - - -def build_pattern(mapping=MAPPING): - mod_list = ' | '.join(["module_name='%s'" % key for key in mapping]) - bare_names = alternates(mapping.keys()) - - yield """name_import=import_name< 'import' ((%s) | - multiple_imports=dotted_as_names< any* (%s) any* >) > - """ % (mod_list, mod_list) - yield """import_from< 'from' (%s) 'import' ['('] - ( any | import_as_name< any 'as' any > | - import_as_names< any* >) [')'] > - """ % mod_list - yield """import_name< 'import' (dotted_as_name< (%s) 'as' any > | - multiple_imports=dotted_as_names< - any* dotted_as_name< (%s) 'as' any > any* >) > - """ % (mod_list, mod_list) - - # Find usages of module members in code e.g. thread.foo(bar) - yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names - - -class FixImports(fixer_base.BaseFix): - - BM_compatible = True - keep_line_order = True - # This is overridden in fix_imports2. - mapping = MAPPING - - # We want to run this fixer late, so fix_import doesn't try to make stdlib - # renames into relative imports. - run_order = 6 - - def build_pattern(self): - return "|".join(build_pattern(self.mapping)) - - def compile_pattern(self): - # We override this, so MAPPING can be pragmatically altered and the - # changes will be reflected in PATTERN. - self.PATTERN = self.build_pattern() - super(FixImports, self).compile_pattern() - - # Don't match the node if it's within another match. - def match(self, node): - match = super(FixImports, self).match - results = match(node) - if results: - # Module usage could be in the trailer of an attribute lookup, so we - # might have nested matches when "bare_with_attr" is present. - if "bare_with_attr" not in results and \ - any(match(obj) for obj in attr_chain(node, "parent")): - return False - return results - return False - - def start_tree(self, tree, filename): - super(FixImports, self).start_tree(tree, filename) - self.replace = {} - - def transform(self, node, results): - import_mod = results.get("module_name") - if import_mod: - mod_name = import_mod.value - new_name = self.mapping[mod_name] - import_mod.replace(Name(new_name, prefix=import_mod.prefix)) - if "name_import" in results: - # If it's not a "from x import x, y" or "import x as y" import, - # marked its usage to be replaced. - self.replace[mod_name] = new_name - if "multiple_imports" in results: - # This is a nasty hack to fix multiple imports on a line (e.g., - # "import StringIO, urlparse"). The problem is that I can't - # figure out an easy way to make a pattern recognize the keys of - # MAPPING randomly sprinkled in an import statement. - results = self.match(node) - if results: - self.transform(node, results) - else: - # Replace usage of the module. - bare_name = results["bare_with_attr"][0] - new_name = self.replace.get(bare_name.value) - if new_name: - bare_name.replace(Name(new_name, prefix=bare_name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_imports2.py b/Lib/lib2to3/fixes/fix_imports2.py deleted file mode 100644 index 9a33c67b1dc194..00000000000000 --- a/Lib/lib2to3/fixes/fix_imports2.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Fix incompatible imports and module references that must be fixed after -fix_imports.""" -from . import fix_imports - - -MAPPING = { - 'whichdb': 'dbm', - 'anydbm': 'dbm', - } - - -class FixImports2(fix_imports.FixImports): - - run_order = 7 - - mapping = MAPPING diff --git a/Lib/lib2to3/fixes/fix_input.py b/Lib/lib2to3/fixes/fix_input.py deleted file mode 100644 index 9cf9a48c471f35..00000000000000 --- a/Lib/lib2to3/fixes/fix_input.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Fixer that changes input(...) into eval(input(...)).""" -# Author: Andre Roberge - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Name -from .. import patcomp - - -context = patcomp.compile_pattern("power< 'eval' trailer< '(' any ')' > >") - - -class FixInput(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< 'input' args=trailer< '(' [any] ')' > > - """ - - def transform(self, node, results): - # If we're already wrapped in an eval() call, we're done. - if context.match(node.parent.parent): - return - - new = node.clone() - new.prefix = "" - return Call(Name("eval"), [new], prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py deleted file mode 100644 index d752843092aacd..00000000000000 --- a/Lib/lib2to3/fixes/fix_intern.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2006 Georg Brandl. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for intern(). - -intern(s) -> sys.intern(s)""" - -# Local imports -from .. import fixer_base -from ..fixer_util import ImportAndCall, touch_import - - -class FixIntern(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'intern' - trailer< lpar='(' - ( not(arglist | argument) any ','> ) - rpar=')' > - after=any* - > - """ - - def transform(self, node, results): - if results: - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - obj = results['obj'] - if obj: - if (obj.type == self.syms.argument and - obj.children[0].value in {'**', '*'}): - return # Make no change. - names = ('sys', 'intern') - new = ImportAndCall(node, results, names) - touch_import(None, 'sys', node) - return new diff --git a/Lib/lib2to3/fixes/fix_isinstance.py b/Lib/lib2to3/fixes/fix_isinstance.py deleted file mode 100644 index bebb1de120424b..00000000000000 --- a/Lib/lib2to3/fixes/fix_isinstance.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2008 Armin Ronacher. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that cleans up a tuple argument to isinstance after the tokens -in it were fixed. This is mainly used to remove double occurrences of -tokens as a leftover of the long -> int / unicode -> str conversion. - -eg. isinstance(x, (int, long)) -> isinstance(x, (int, int)) - -> isinstance(x, int) -""" - -from .. import fixer_base -from ..fixer_util import token - - -class FixIsinstance(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< - 'isinstance' - trailer< '(' arglist< any ',' atom< '(' - args=testlist_gexp< any+ > - ')' > > ')' > - > - """ - - run_order = 6 - - def transform(self, node, results): - names_inserted = set() - testlist = results["args"] - args = testlist.children - new_args = [] - iterator = enumerate(args) - for idx, arg in iterator: - if arg.type == token.NAME and arg.value in names_inserted: - if idx < len(args) - 1 and args[idx + 1].type == token.COMMA: - next(iterator) - continue - else: - new_args.append(arg) - if arg.type == token.NAME: - names_inserted.add(arg.value) - if new_args and new_args[-1].type == token.COMMA: - del new_args[-1] - if len(new_args) == 1: - atom = testlist.parent - new_args[0].prefix = atom.prefix - atom.replace(new_args[0]) - else: - args[:] = new_args - node.changed() diff --git a/Lib/lib2to3/fixes/fix_itertools.py b/Lib/lib2to3/fixes/fix_itertools.py deleted file mode 100644 index 8e78d6c689f439..00000000000000 --- a/Lib/lib2to3/fixes/fix_itertools.py +++ /dev/null @@ -1,43 +0,0 @@ -""" Fixer for itertools.(imap|ifilter|izip) --> (map|filter|zip) and - itertools.ifilterfalse --> itertools.filterfalse (bugs 2360-2363) - - imports from itertools are fixed in fix_itertools_import.py - - If itertools is imported as something else (ie: import itertools as it; - it.izip(spam, eggs)) method calls will not get fixed. - """ - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixItertools(fixer_base.BaseFix): - BM_compatible = True - it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')" - PATTERN = """ - power< it='itertools' - trailer< - dot='.' func=%(it_funcs)s > trailer< '(' [any] ')' > > - | - power< func=%(it_funcs)s trailer< '(' [any] ')' > > - """ %(locals()) - - # Needs to be run after fix_(map|zip|filter) - run_order = 6 - - def transform(self, node, results): - prefix = None - func = results['func'][0] - if ('it' in results and - func.value not in ('ifilterfalse', 'izip_longest')): - dot, it = (results['dot'], results['it']) - # Remove the 'itertools' - prefix = it.prefix - it.remove() - # Replace the node which contains ('.', 'function') with the - # function (to be consistent with the second part of the pattern) - dot.remove() - func.parent.replace(func) - - prefix = prefix or func.prefix - func.replace(Name(func.value[1:], prefix=prefix)) diff --git a/Lib/lib2to3/fixes/fix_itertools_imports.py b/Lib/lib2to3/fixes/fix_itertools_imports.py deleted file mode 100644 index 0ddbc7b8422991..00000000000000 --- a/Lib/lib2to3/fixes/fix_itertools_imports.py +++ /dev/null @@ -1,57 +0,0 @@ -""" Fixer for imports of itertools.(imap|ifilter|izip|ifilterfalse) """ - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import BlankLine, syms, token - - -class FixItertoolsImports(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - import_from< 'from' 'itertools' 'import' imports=any > - """ %(locals()) - - def transform(self, node, results): - imports = results['imports'] - if imports.type == syms.import_as_name or not imports.children: - children = [imports] - else: - children = imports.children - for child in children[::2]: - if child.type == token.NAME: - member = child.value - name_node = child - elif child.type == token.STAR: - # Just leave the import as is. - return - else: - assert child.type == syms.import_as_name - name_node = child.children[0] - member_name = name_node.value - if member_name in ('imap', 'izip', 'ifilter'): - child.value = None - child.remove() - elif member_name in ('ifilterfalse', 'izip_longest'): - node.changed() - name_node.value = ('filterfalse' if member_name[1] == 'f' - else 'zip_longest') - - # Make sure the import statement is still sane - children = imports.children[:] or [imports] - remove_comma = True - for child in children: - if remove_comma and child.type == token.COMMA: - child.remove() - else: - remove_comma ^= True - - while children and children[-1].type == token.COMMA: - children.pop().remove() - - # If there are no imports left, just get rid of the entire statement - if (not (imports.children or getattr(imports, 'value', None)) or - imports.parent is None): - p = node.prefix - node = BlankLine() - node.prefix = p - return node diff --git a/Lib/lib2to3/fixes/fix_long.py b/Lib/lib2to3/fixes/fix_long.py deleted file mode 100644 index f227c9f4981538..00000000000000 --- a/Lib/lib2to3/fixes/fix_long.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that turns 'long' into 'int' everywhere. -""" - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import is_probably_builtin - - -class FixLong(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "'long'" - - def transform(self, node, results): - if is_probably_builtin(node): - node.value = "int" - node.changed() diff --git a/Lib/lib2to3/fixes/fix_map.py b/Lib/lib2to3/fixes/fix_map.py deleted file mode 100644 index 78cf81c6f94098..00000000000000 --- a/Lib/lib2to3/fixes/fix_map.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes map(F, ...) into list(map(F, ...)) unless there -exists a 'from future_builtins import map' statement in the top-level -namespace. - -As a special case, map(None, X) is changed into list(X). (This is -necessary because the semantics are changed in this case -- the new -map(None, X) is equivalent to [(x,) for x in X].) - -We avoid the transformation (except for the special case mentioned -above) if the map() call is directly contained in iter(<>), list(<>), -tuple(<>), sorted(<>), ...join(<>), or for V in <>:. - -NOTE: This is still not correct if the original code was depending on -map(F, X, Y, ...) to go on until the longest argument is exhausted, -substituting None for missing values -- like zip(), it now stops as -soon as the shortest argument is exhausted. -""" - -# Local imports -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, ArgList, Call, ListComp, in_special_context -from ..pygram import python_symbols as syms -from ..pytree import Node - - -class FixMap(fixer_base.ConditionalFix): - BM_compatible = True - - PATTERN = """ - map_none=power< - 'map' - trailer< '(' arglist< 'None' ',' arg=any [','] > ')' > - [extra_trailers=trailer*] - > - | - map_lambda=power< - 'map' - trailer< - '(' - arglist< - lambdef< 'lambda' - (fp=NAME | vfpdef< '(' fp=NAME ')'> ) ':' xp=any - > - ',' - it=any - > - ')' - > - [extra_trailers=trailer*] - > - | - power< - 'map' args=trailer< '(' [any] ')' > - [extra_trailers=trailer*] - > - """ - - skip_on = 'future_builtins.map' - - def transform(self, node, results): - if self.should_skip(node): - return - - trailers = [] - if 'extra_trailers' in results: - for t in results['extra_trailers']: - trailers.append(t.clone()) - - if node.parent.type == syms.simple_stmt: - self.warning(node, "You should use a for loop here") - new = node.clone() - new.prefix = "" - new = Call(Name("list"), [new]) - elif "map_lambda" in results: - new = ListComp(results["xp"].clone(), - results["fp"].clone(), - results["it"].clone()) - new = Node(syms.power, [new] + trailers, prefix="") - - else: - if "map_none" in results: - new = results["arg"].clone() - new.prefix = "" - else: - if "args" in results: - args = results["args"] - if args.type == syms.trailer and \ - args.children[1].type == syms.arglist and \ - args.children[1].children[0].type == token.NAME and \ - args.children[1].children[0].value == "None": - self.warning(node, "cannot convert map(None, ...) " - "with multiple arguments because map() " - "now truncates to the shortest sequence") - return - - new = Node(syms.power, [Name("map"), args.clone()]) - new.prefix = "" - - if in_special_context(node): - return None - - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = "" - - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py deleted file mode 100644 index fe547b2228072a..00000000000000 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Fixer for __metaclass__ = X -> (metaclass=X) methods. - - The various forms of classef (inherits nothing, inherits once, inherits - many) don't parse the same in the CST so we look at ALL classes for - a __metaclass__ and if we find one normalize the inherits to all be - an arglist. - - For one-liner classes ('class X: pass') there is no indent/dedent so - we normalize those into having a suite. - - Moving the __metaclass__ into the classdef can also cause the class - body to be empty so there is some special casing for that as well. - - This fixer also tries very hard to keep original indenting and spacing - in all those corner cases. - -""" -# Author: Jack Diederich - -# Local imports -from .. import fixer_base -from ..pygram import token -from ..fixer_util import syms, Node, Leaf - - -def has_metaclass(parent): - """ we have to check the cls_node without changing it. - There are two possibilities: - 1) clsdef => suite => simple_stmt => expr_stmt => Leaf('__meta') - 2) clsdef => simple_stmt => expr_stmt => Leaf('__meta') - """ - for node in parent.children: - if node.type == syms.suite: - return has_metaclass(node) - elif node.type == syms.simple_stmt and node.children: - expr_node = node.children[0] - if expr_node.type == syms.expr_stmt and expr_node.children: - left_side = expr_node.children[0] - if isinstance(left_side, Leaf) and \ - left_side.value == '__metaclass__': - return True - return False - - -def fixup_parse_tree(cls_node): - """ one-line classes don't get a suite in the parse tree so we add - one to normalize the tree - """ - for node in cls_node.children: - if node.type == syms.suite: - # already in the preferred format, do nothing - return - - # !%@#! one-liners have no suite node, we have to fake one up - for i, node in enumerate(cls_node.children): - if node.type == token.COLON: - break - else: - raise ValueError("No class suite and no ':'!") - - # move everything into a suite node - suite = Node(syms.suite, []) - while cls_node.children[i+1:]: - move_node = cls_node.children[i+1] - suite.append_child(move_node.clone()) - move_node.remove() - cls_node.append_child(suite) - node = suite - - -def fixup_simple_stmt(parent, i, stmt_node): - """ if there is a semi-colon all the parts count as part of the same - simple_stmt. We just want the __metaclass__ part so we move - everything after the semi-colon into its own simple_stmt node - """ - for semi_ind, node in enumerate(stmt_node.children): - if node.type == token.SEMI: # *sigh* - break - else: - return - - node.remove() # kill the semicolon - new_expr = Node(syms.expr_stmt, []) - new_stmt = Node(syms.simple_stmt, [new_expr]) - while stmt_node.children[semi_ind:]: - move_node = stmt_node.children[semi_ind] - new_expr.append_child(move_node.clone()) - move_node.remove() - parent.insert_child(i, new_stmt) - new_leaf1 = new_stmt.children[0].children[0] - old_leaf1 = stmt_node.children[0].children[0] - new_leaf1.prefix = old_leaf1.prefix - - -def remove_trailing_newline(node): - if node.children and node.children[-1].type == token.NEWLINE: - node.children[-1].remove() - - -def find_metas(cls_node): - # find the suite node (Mmm, sweet nodes) - for node in cls_node.children: - if node.type == syms.suite: - break - else: - raise ValueError("No class suite!") - - # look for simple_stmt[ expr_stmt[ Leaf('__metaclass__') ] ] - for i, simple_node in list(enumerate(node.children)): - if simple_node.type == syms.simple_stmt and simple_node.children: - expr_node = simple_node.children[0] - if expr_node.type == syms.expr_stmt and expr_node.children: - # Check if the expr_node is a simple assignment. - left_node = expr_node.children[0] - if isinstance(left_node, Leaf) and \ - left_node.value == '__metaclass__': - # We found an assignment to __metaclass__. - fixup_simple_stmt(node, i, simple_node) - remove_trailing_newline(simple_node) - yield (node, i, simple_node) - - -def fixup_indent(suite): - """ If an INDENT is followed by a thing with a prefix then nuke the prefix - Otherwise we get in trouble when removing __metaclass__ at suite start - """ - kids = suite.children[::-1] - # find the first indent - while kids: - node = kids.pop() - if node.type == token.INDENT: - break - - # find the first Leaf - while kids: - node = kids.pop() - if isinstance(node, Leaf) and node.type != token.DEDENT: - if node.prefix: - node.prefix = '' - return - else: - kids.extend(node.children[::-1]) - - -class FixMetaclass(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - classdef - """ - - def transform(self, node, results): - if not has_metaclass(node): - return - - fixup_parse_tree(node) - - # find metaclasses, keep the last one - last_metaclass = None - for suite, i, stmt in find_metas(node): - last_metaclass = stmt - stmt.remove() - - text_type = node.children[0].type # always Leaf(nnn, 'class') - - # figure out what kind of classdef we have - if len(node.children) == 7: - # Node(classdef, ['class', 'name', '(', arglist, ')', ':', suite]) - # 0 1 2 3 4 5 6 - if node.children[3].type == syms.arglist: - arglist = node.children[3] - # Node(classdef, ['class', 'name', '(', 'Parent', ')', ':', suite]) - else: - parent = node.children[3].clone() - arglist = Node(syms.arglist, [parent]) - node.set_child(3, arglist) - elif len(node.children) == 6: - # Node(classdef, ['class', 'name', '(', ')', ':', suite]) - # 0 1 2 3 4 5 - arglist = Node(syms.arglist, []) - node.insert_child(3, arglist) - elif len(node.children) == 4: - # Node(classdef, ['class', 'name', ':', suite]) - # 0 1 2 3 - arglist = Node(syms.arglist, []) - node.insert_child(2, Leaf(token.RPAR, ')')) - node.insert_child(2, arglist) - node.insert_child(2, Leaf(token.LPAR, '(')) - else: - raise ValueError("Unexpected class definition") - - # now stick the metaclass in the arglist - meta_txt = last_metaclass.children[0].children[0] - meta_txt.value = 'metaclass' - orig_meta_prefix = meta_txt.prefix - - if arglist.children: - arglist.append_child(Leaf(token.COMMA, ',')) - meta_txt.prefix = ' ' - else: - meta_txt.prefix = '' - - # compact the expression "metaclass = Meta" -> "metaclass=Meta" - expr_stmt = last_metaclass.children[0] - assert expr_stmt.type == syms.expr_stmt - expr_stmt.children[1].prefix = '' - expr_stmt.children[2].prefix = '' - - arglist.append_child(last_metaclass) - - fixup_indent(suite) - - # check for empty suite - if not suite.children: - # one-liner that was just __metaclass_ - suite.remove() - pass_leaf = Leaf(text_type, 'pass') - pass_leaf.prefix = orig_meta_prefix - node.append_child(pass_leaf) - node.append_child(Leaf(token.NEWLINE, '\n')) - - elif len(suite.children) > 1 and \ - (suite.children[-2].type == token.INDENT and - suite.children[-1].type == token.DEDENT): - # there was only one line in the class body and it was __metaclass__ - pass_leaf = Leaf(text_type, 'pass') - suite.insert_child(-1, pass_leaf) - suite.insert_child(-1, Leaf(token.NEWLINE, '\n')) diff --git a/Lib/lib2to3/fixes/fix_methodattrs.py b/Lib/lib2to3/fixes/fix_methodattrs.py deleted file mode 100644 index 7f9004f00e6e8f..00000000000000 --- a/Lib/lib2to3/fixes/fix_methodattrs.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Fix bound method attributes (method.im_? -> method.__?__). -""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -MAP = { - "im_func" : "__func__", - "im_self" : "__self__", - "im_class" : "__self__.__class__" - } - -class FixMethodattrs(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< any+ trailer< '.' attr=('im_func' | 'im_self' | 'im_class') > any* > - """ - - def transform(self, node, results): - attr = results["attr"][0] - new = MAP[attr.value] - attr.replace(Name(new, prefix=attr.prefix)) diff --git a/Lib/lib2to3/fixes/fix_ne.py b/Lib/lib2to3/fixes/fix_ne.py deleted file mode 100644 index e3ee10f4a63e0c..00000000000000 --- a/Lib/lib2to3/fixes/fix_ne.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that turns <> into !=.""" - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base - - -class FixNe(fixer_base.BaseFix): - # This is so simple that we don't need the pattern compiler. - - _accept_type = token.NOTEQUAL - - def match(self, node): - # Override - return node.value == "<>" - - def transform(self, node, results): - new = pytree.Leaf(token.NOTEQUAL, "!=", prefix=node.prefix) - return new diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py deleted file mode 100644 index 9f6305e1d49dc5..00000000000000 --- a/Lib/lib2to3/fixes/fix_next.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Fixer for it.next() -> next(it), per PEP 3114.""" -# Author: Collin Winter - -# Things that currently aren't covered: -# - listcomp "next" names aren't warned -# - "with" statement targets aren't checked - -# Local imports -from ..pgen2 import token -from ..pygram import python_symbols as syms -from .. import fixer_base -from ..fixer_util import Name, Call, find_binding - -bind_warning = "Calls to builtin next() possibly shadowed by global binding" - - -class FixNext(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< base=any+ trailer< '.' attr='next' > trailer< '(' ')' > > - | - power< head=any+ trailer< '.' attr='next' > not trailer< '(' ')' > > - | - classdef< 'class' any+ ':' - suite< any* - funcdef< 'def' - name='next' - parameters< '(' NAME ')' > any+ > - any* > > - | - global=global_stmt< 'global' any* 'next' any* > - """ - - order = "pre" # Pre-order tree traversal - - def start_tree(self, tree, filename): - super(FixNext, self).start_tree(tree, filename) - - n = find_binding('next', tree) - if n: - self.warning(n, bind_warning) - self.shadowed_next = True - else: - self.shadowed_next = False - - def transform(self, node, results): - assert results - - base = results.get("base") - attr = results.get("attr") - name = results.get("name") - - if base: - if self.shadowed_next: - attr.replace(Name("__next__", prefix=attr.prefix)) - else: - base = [n.clone() for n in base] - base[0].prefix = "" - node.replace(Call(Name("next", prefix=node.prefix), base)) - elif name: - n = Name("__next__", prefix=name.prefix) - name.replace(n) - elif attr: - # We don't do this transformation if we're assigning to "x.next". - # Unfortunately, it doesn't seem possible to do this in PATTERN, - # so it's being done here. - if is_assign_target(node): - head = results["head"] - if "".join([str(n) for n in head]).strip() == '__builtin__': - self.warning(node, bind_warning) - return - attr.replace(Name("__next__")) - elif "global" in results: - self.warning(node, bind_warning) - self.shadowed_next = True - - -### The following functions help test if node is part of an assignment -### target. - -def is_assign_target(node): - assign = find_assign(node) - if assign is None: - return False - - for child in assign.children: - if child.type == token.EQUAL: - return False - elif is_subtree(child, node): - return True - return False - -def find_assign(node): - if node.type == syms.expr_stmt: - return node - if node.type == syms.simple_stmt or node.parent is None: - return None - return find_assign(node.parent) - -def is_subtree(root, node): - if root == node: - return True - return any(is_subtree(c, node) for c in root.children) diff --git a/Lib/lib2to3/fixes/fix_nonzero.py b/Lib/lib2to3/fixes/fix_nonzero.py deleted file mode 100644 index c2295969a7728f..00000000000000 --- a/Lib/lib2to3/fixes/fix_nonzero.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Fixer for __nonzero__ -> __bool__ methods.""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixNonzero(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - classdef< 'class' any+ ':' - suite< any* - funcdef< 'def' name='__nonzero__' - parameters< '(' NAME ')' > any+ > - any* > > - """ - - def transform(self, node, results): - name = results["name"] - new = Name("__bool__", prefix=name.prefix) - name.replace(new) diff --git a/Lib/lib2to3/fixes/fix_numliterals.py b/Lib/lib2to3/fixes/fix_numliterals.py deleted file mode 100644 index 79207d4aa368ae..00000000000000 --- a/Lib/lib2to3/fixes/fix_numliterals.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Fixer that turns 1L into 1, 0755 into 0o755. -""" -# Copyright 2007 Georg Brandl. -# Licensed to PSF under a Contributor Agreement. - -# Local imports -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Number - - -class FixNumliterals(fixer_base.BaseFix): - # This is so simple that we don't need the pattern compiler. - - _accept_type = token.NUMBER - - def match(self, node): - # Override - return (node.value.startswith("0") or node.value[-1] in "Ll") - - def transform(self, node, results): - val = node.value - if val[-1] in 'Ll': - val = val[:-1] - elif val.startswith('0') and val.isdigit() and len(set(val)) > 1: - val = "0o" + val[1:] - - return Number(val, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_operator.py b/Lib/lib2to3/fixes/fix_operator.py deleted file mode 100644 index d303cd2018befb..00000000000000 --- a/Lib/lib2to3/fixes/fix_operator.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Fixer for operator functions. - -operator.isCallable(obj) -> callable(obj) -operator.sequenceIncludes(obj) -> operator.contains(obj) -operator.isSequenceType(obj) -> isinstance(obj, collections.abc.Sequence) -operator.isMappingType(obj) -> isinstance(obj, collections.abc.Mapping) -operator.isNumberType(obj) -> isinstance(obj, numbers.Number) -operator.repeat(obj, n) -> operator.mul(obj, n) -operator.irepeat(obj, n) -> operator.imul(obj, n) -""" - -import collections.abc - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import Call, Name, String, touch_import - - -def invocation(s): - def dec(f): - f.invocation = s - return f - return dec - - -class FixOperator(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - methods = """ - method=('isCallable'|'sequenceIncludes' - |'isSequenceType'|'isMappingType'|'isNumberType' - |'repeat'|'irepeat') - """ - obj = "'(' obj=any ')'" - PATTERN = """ - power< module='operator' - trailer< '.' %(methods)s > trailer< %(obj)s > > - | - power< %(methods)s trailer< %(obj)s > > - """ % dict(methods=methods, obj=obj) - - def transform(self, node, results): - method = self._check_method(node, results) - if method is not None: - return method(node, results) - - @invocation("operator.contains(%s)") - def _sequenceIncludes(self, node, results): - return self._handle_rename(node, results, "contains") - - @invocation("callable(%s)") - def _isCallable(self, node, results): - obj = results["obj"] - return Call(Name("callable"), [obj.clone()], prefix=node.prefix) - - @invocation("operator.mul(%s)") - def _repeat(self, node, results): - return self._handle_rename(node, results, "mul") - - @invocation("operator.imul(%s)") - def _irepeat(self, node, results): - return self._handle_rename(node, results, "imul") - - @invocation("isinstance(%s, collections.abc.Sequence)") - def _isSequenceType(self, node, results): - return self._handle_type2abc(node, results, "collections.abc", "Sequence") - - @invocation("isinstance(%s, collections.abc.Mapping)") - def _isMappingType(self, node, results): - return self._handle_type2abc(node, results, "collections.abc", "Mapping") - - @invocation("isinstance(%s, numbers.Number)") - def _isNumberType(self, node, results): - return self._handle_type2abc(node, results, "numbers", "Number") - - def _handle_rename(self, node, results, name): - method = results["method"][0] - method.value = name - method.changed() - - def _handle_type2abc(self, node, results, module, abc): - touch_import(None, module, node) - obj = results["obj"] - args = [obj.clone(), String(", " + ".".join([module, abc]))] - return Call(Name("isinstance"), args, prefix=node.prefix) - - def _check_method(self, node, results): - method = getattr(self, "_" + results["method"][0].value) - if isinstance(method, collections.abc.Callable): - if "module" in results: - return method - else: - sub = (str(results["obj"]),) - invocation_str = method.invocation % sub - self.warning(node, "You should use '%s' here." % invocation_str) - return None diff --git a/Lib/lib2to3/fixes/fix_paren.py b/Lib/lib2to3/fixes/fix_paren.py deleted file mode 100644 index df3da5f5232c9c..00000000000000 --- a/Lib/lib2to3/fixes/fix_paren.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Fixer that adds parentheses where they are required - -This converts ``[x for x in 1, 2]`` to ``[x for x in (1, 2)]``.""" - -# By Taek Joo Kim and Benjamin Peterson - -# Local imports -from .. import fixer_base -from ..fixer_util import LParen, RParen - -# XXX This doesn't support nested for loops like [x for x in 1, 2 for x in 1, 2] -class FixParen(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - atom< ('[' | '(') - (listmaker< any - comp_for< - 'for' NAME 'in' - target=testlist_safe< any (',' any)+ [','] - > - [any] - > - > - | - testlist_gexp< any - comp_for< - 'for' NAME 'in' - target=testlist_safe< any (',' any)+ [','] - > - [any] - > - >) - (']' | ')') > - """ - - def transform(self, node, results): - target = results["target"] - - lparen = LParen() - lparen.prefix = target.prefix - target.prefix = "" # Make it hug the parentheses - target.insert_child(0, lparen) - target.append_child(RParen()) diff --git a/Lib/lib2to3/fixes/fix_print.py b/Lib/lib2to3/fixes/fix_print.py deleted file mode 100644 index 8780322265f6fe..00000000000000 --- a/Lib/lib2to3/fixes/fix_print.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for print. - -Change: - 'print' into 'print()' - 'print ...' into 'print(...)' - 'print ... ,' into 'print(..., end=" ")' - 'print >>x, ...' into 'print(..., file=x)' - -No changes are applied if print_function is imported from __future__ - -""" - -# Local imports -from .. import patcomp -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, Comma, String - - -parend_expr = patcomp.compile_pattern( - """atom< '(' [atom|STRING|NAME] ')' >""" - ) - - -class FixPrint(fixer_base.BaseFix): - - BM_compatible = True - - PATTERN = """ - simple_stmt< any* bare='print' any* > | print_stmt - """ - - def transform(self, node, results): - assert results - - bare_print = results.get("bare") - - if bare_print: - # Special-case print all by itself - bare_print.replace(Call(Name("print"), [], - prefix=bare_print.prefix)) - return - assert node.children[0] == Name("print") - args = node.children[1:] - if len(args) == 1 and parend_expr.match(args[0]): - # We don't want to keep sticking parens around an - # already-parenthesised expression. - return - - sep = end = file = None - if args and args[-1] == Comma(): - args = args[:-1] - end = " " - if args and args[0] == pytree.Leaf(token.RIGHTSHIFT, ">>"): - assert len(args) >= 2 - file = args[1].clone() - args = args[3:] # Strip a possible comma after the file expression - # Now synthesize a print(args, sep=..., end=..., file=...) node. - l_args = [arg.clone() for arg in args] - if l_args: - l_args[0].prefix = "" - if sep is not None or end is not None or file is not None: - if sep is not None: - self.add_kwarg(l_args, "sep", String(repr(sep))) - if end is not None: - self.add_kwarg(l_args, "end", String(repr(end))) - if file is not None: - self.add_kwarg(l_args, "file", file) - n_stmt = Call(Name("print"), l_args) - n_stmt.prefix = node.prefix - return n_stmt - - def add_kwarg(self, l_nodes, s_kwd, n_expr): - # XXX All this prefix-setting may lose comments (though rarely) - n_expr.prefix = "" - n_argument = pytree.Node(self.syms.argument, - (Name(s_kwd), - pytree.Leaf(token.EQUAL, "="), - n_expr)) - if l_nodes: - l_nodes.append(Comma()) - n_argument.prefix = " " - l_nodes.append(n_argument) diff --git a/Lib/lib2to3/fixes/fix_raise.py b/Lib/lib2to3/fixes/fix_raise.py deleted file mode 100644 index 05aa21e74a30ff..00000000000000 --- a/Lib/lib2to3/fixes/fix_raise.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Fixer for 'raise E, V, T' - -raise -> raise -raise E -> raise E -raise E, V -> raise E(V) -raise E, V, T -> raise E(V).with_traceback(T) -raise E, None, T -> raise E.with_traceback(T) - -raise (((E, E'), E''), E'''), V -> raise E(V) -raise "foo", V, T -> warns about string exceptions - - -CAVEATS: -1) "raise E, V" will be incorrectly translated if V is an exception - instance. The correct Python 3 idiom is - - raise E from V - - but since we can't detect instance-hood by syntax alone and since - any client code would have to be changed as well, we don't automate - this. -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, Attr, ArgList, is_tuple - -class FixRaise(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - raise_stmt< 'raise' exc=any [',' val=any [',' tb=any]] > - """ - - def transform(self, node, results): - syms = self.syms - - exc = results["exc"].clone() - if exc.type == token.STRING: - msg = "Python 3 does not support string exceptions" - self.cannot_convert(node, msg) - return - - # Python 2 supports - # raise ((((E1, E2), E3), E4), E5), V - # as a synonym for - # raise E1, V - # Since Python 3 will not support this, we recurse down any tuple - # literals, always taking the first element. - if is_tuple(exc): - while is_tuple(exc): - # exc.children[1:-1] is the unparenthesized tuple - # exc.children[1].children[0] is the first element of the tuple - exc = exc.children[1].children[0].clone() - exc.prefix = " " - - if "val" not in results: - # One-argument raise - new = pytree.Node(syms.raise_stmt, [Name("raise"), exc]) - new.prefix = node.prefix - return new - - val = results["val"].clone() - if is_tuple(val): - args = [c.clone() for c in val.children[1:-1]] - else: - val.prefix = "" - args = [val] - - if "tb" in results: - tb = results["tb"].clone() - tb.prefix = "" - - e = exc - # If there's a traceback and None is passed as the value, then don't - # add a call, since the user probably just wants to add a - # traceback. See issue #9661. - if val.type != token.NAME or val.value != "None": - e = Call(exc, args) - with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])] - new = pytree.Node(syms.simple_stmt, [Name("raise")] + with_tb) - new.prefix = node.prefix - return new - else: - return pytree.Node(syms.raise_stmt, - [Name("raise"), Call(exc, args)], - prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_raw_input.py b/Lib/lib2to3/fixes/fix_raw_input.py deleted file mode 100644 index a51bb694b9e01e..00000000000000 --- a/Lib/lib2to3/fixes/fix_raw_input.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Fixer that changes raw_input(...) into input(...).""" -# Author: Andre Roberge - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixRawInput(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - power< name='raw_input' trailer< '(' [any] ')' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("input", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_reduce.py b/Lib/lib2to3/fixes/fix_reduce.py deleted file mode 100644 index 00e5aa1c33d482..00000000000000 --- a/Lib/lib2to3/fixes/fix_reduce.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2008 Armin Ronacher. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for reduce(). - -Makes sure reduce() is imported from the functools module if reduce is -used in that module. -""" - -from lib2to3 import fixer_base -from lib2to3.fixer_util import touch_import - - - -class FixReduce(fixer_base.BaseFix): - - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'reduce' - trailer< '(' - arglist< ( - (not(argument) any ',' - not(argument - > - """ - - def transform(self, node, results): - touch_import('functools', 'reduce', node) diff --git a/Lib/lib2to3/fixes/fix_reload.py b/Lib/lib2to3/fixes/fix_reload.py deleted file mode 100644 index b30841131c51f9..00000000000000 --- a/Lib/lib2to3/fixes/fix_reload.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Fixer for reload(). - -reload(s) -> importlib.reload(s)""" - -# Local imports -from .. import fixer_base -from ..fixer_util import ImportAndCall, touch_import - - -class FixReload(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'reload' - trailer< lpar='(' - ( not(arglist | argument) any ','> ) - rpar=')' > - after=any* - > - """ - - def transform(self, node, results): - if results: - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - obj = results['obj'] - if obj: - if (obj.type == self.syms.argument and - obj.children[0].value in {'**', '*'}): - return # Make no change. - names = ('importlib', 'reload') - new = ImportAndCall(node, results, names) - touch_import(None, 'importlib', node) - return new diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py deleted file mode 100644 index c0e3705ab7be19..00000000000000 --- a/Lib/lib2to3/fixes/fix_renames.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Fix incompatible renames - -Fixes: - * sys.maxint -> sys.maxsize -""" -# Author: Christian Heimes -# based on Collin Winter's fix_import - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, attr_chain - -MAPPING = {"sys": {"maxint" : "maxsize"}, - } -LOOKUP = {} - -def alternates(members): - return "(" + "|".join(map(repr, members)) + ")" - - -def build_pattern(): - #bare = set() - for module, replace in list(MAPPING.items()): - for old_attr, new_attr in list(replace.items()): - LOOKUP[(module, old_attr)] = new_attr - #bare.add(module) - #bare.add(old_attr) - #yield """ - # import_name< 'import' (module=%r - # | dotted_as_names< any* module=%r any* >) > - # """ % (module, module) - yield """ - import_from< 'from' module_name=%r 'import' - ( attr_name=%r | import_as_name< attr_name=%r 'as' any >) > - """ % (module, old_attr, old_attr) - yield """ - power< module_name=%r trailer< '.' attr_name=%r > any* > - """ % (module, old_attr) - #yield """bare_name=%s""" % alternates(bare) - - -class FixRenames(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "|".join(build_pattern()) - - order = "pre" # Pre-order tree traversal - - # Don't match the node if it's within another match - def match(self, node): - match = super(FixRenames, self).match - results = match(node) - if results: - if any(match(obj) for obj in attr_chain(node, "parent")): - return False - return results - return False - - #def start_tree(self, tree, filename): - # super(FixRenames, self).start_tree(tree, filename) - # self.replace = {} - - def transform(self, node, results): - mod_name = results.get("module_name") - attr_name = results.get("attr_name") - #bare_name = results.get("bare_name") - #import_mod = results.get("module") - - if mod_name and attr_name: - new_attr = LOOKUP[(mod_name.value, attr_name.value)] - attr_name.replace(Name(new_attr, prefix=attr_name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_repr.py b/Lib/lib2to3/fixes/fix_repr.py deleted file mode 100644 index 1150bb8b9db2af..00000000000000 --- a/Lib/lib2to3/fixes/fix_repr.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that transforms `xyzzy` into repr(xyzzy).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Name, parenthesize - - -class FixRepr(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - atom < '`' expr=any '`' > - """ - - def transform(self, node, results): - expr = results["expr"].clone() - - if expr.type == self.syms.testlist1: - expr = parenthesize(expr) - return Call(Name("repr"), [expr], prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_set_literal.py b/Lib/lib2to3/fixes/fix_set_literal.py deleted file mode 100644 index 762550cf73dc0b..00000000000000 --- a/Lib/lib2to3/fixes/fix_set_literal.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Optional fixer to transform set() calls to set literals. -""" - -# Author: Benjamin Peterson - -from lib2to3 import fixer_base, pytree -from lib2to3.fixer_util import token, syms - - - -class FixSetLiteral(fixer_base.BaseFix): - - BM_compatible = True - explicit = True - - PATTERN = """power< 'set' trailer< '(' - (atom=atom< '[' (items=listmaker< any ((',' any)* [',']) > - | - single=any) ']' > - | - atom< '(' items=testlist_gexp< any ((',' any)* [',']) > ')' > - ) - ')' > > - """ - - def transform(self, node, results): - single = results.get("single") - if single: - # Make a fake listmaker - fake = pytree.Node(syms.listmaker, [single.clone()]) - single.replace(fake) - items = fake - else: - items = results["items"] - - # Build the contents of the literal - literal = [pytree.Leaf(token.LBRACE, "{")] - literal.extend(n.clone() for n in items.children) - literal.append(pytree.Leaf(token.RBRACE, "}")) - # Set the prefix of the right brace to that of the ')' or ']' - literal[-1].prefix = items.next_sibling.prefix - maker = pytree.Node(syms.dictsetmaker, literal) - maker.prefix = node.prefix - - # If the original was a one tuple, we need to remove the extra comma. - if len(maker.children) == 4: - n = maker.children[2] - n.remove() - maker.children[-1].prefix = n.prefix - - # Finally, replace the set call with our shiny new literal. - return maker diff --git a/Lib/lib2to3/fixes/fix_standarderror.py b/Lib/lib2to3/fixes/fix_standarderror.py deleted file mode 100644 index dc742167e6e9d4..00000000000000 --- a/Lib/lib2to3/fixes/fix_standarderror.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for StandardError -> Exception.""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixStandarderror(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - 'StandardError' - """ - - def transform(self, node, results): - return Name("Exception", prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_sys_exc.py b/Lib/lib2to3/fixes/fix_sys_exc.py deleted file mode 100644 index f6039690374ab2..00000000000000 --- a/Lib/lib2to3/fixes/fix_sys_exc.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Fixer for sys.exc_{type, value, traceback} - -sys.exc_type -> sys.exc_info()[0] -sys.exc_value -> sys.exc_info()[1] -sys.exc_traceback -> sys.exc_info()[2] -""" - -# By Jeff Balogh and Benjamin Peterson - -# Local imports -from .. import fixer_base -from ..fixer_util import Attr, Call, Name, Number, Subscript, Node, syms - -class FixSysExc(fixer_base.BaseFix): - # This order matches the ordering of sys.exc_info(). - exc_info = ["exc_type", "exc_value", "exc_traceback"] - BM_compatible = True - PATTERN = """ - power< 'sys' trailer< dot='.' attribute=(%s) > > - """ % '|'.join("'%s'" % e for e in exc_info) - - def transform(self, node, results): - sys_attr = results["attribute"][0] - index = Number(self.exc_info.index(sys_attr.value)) - - call = Call(Name("exc_info"), prefix=sys_attr.prefix) - attr = Attr(Name("sys"), call) - attr[1].children[0].prefix = results["dot"].prefix - attr.append(Subscript(index)) - return Node(syms.power, attr, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_throw.py b/Lib/lib2to3/fixes/fix_throw.py deleted file mode 100644 index aac29169b4e98e..00000000000000 --- a/Lib/lib2to3/fixes/fix_throw.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Fixer for generator.throw(E, V, T). - -g.throw(E) -> g.throw(E) -g.throw(E, V) -> g.throw(E(V)) -g.throw(E, V, T) -> g.throw(E(V).with_traceback(T)) - -g.throw("foo"[, V[, T]]) will warn about string exceptions.""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, ArgList, Attr, is_tuple - -class FixThrow(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< any trailer< '.' 'throw' > - trailer< '(' args=arglist< exc=any ',' val=any [',' tb=any] > ')' > - > - | - power< any trailer< '.' 'throw' > trailer< '(' exc=any ')' > > - """ - - def transform(self, node, results): - syms = self.syms - - exc = results["exc"].clone() - if exc.type is token.STRING: - self.cannot_convert(node, "Python 3 does not support string exceptions") - return - - # Leave "g.throw(E)" alone - val = results.get("val") - if val is None: - return - - val = val.clone() - if is_tuple(val): - args = [c.clone() for c in val.children[1:-1]] - else: - val.prefix = "" - args = [val] - - throw_args = results["args"] - - if "tb" in results: - tb = results["tb"].clone() - tb.prefix = "" - - e = Call(exc, args) - with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])] - throw_args.replace(pytree.Node(syms.power, with_tb)) - else: - throw_args.replace(Call(exc, args)) diff --git a/Lib/lib2to3/fixes/fix_tuple_params.py b/Lib/lib2to3/fixes/fix_tuple_params.py deleted file mode 100644 index cad755ffdbefb3..00000000000000 --- a/Lib/lib2to3/fixes/fix_tuple_params.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Fixer for function definitions with tuple parameters. - -def func(((a, b), c), d): - ... - - -> - -def func(x, d): - ((a, b), c) = x - ... - -It will also support lambdas: - - lambda (x, y): x + y -> lambda t: t[0] + t[1] - - # The parens are a syntax error in Python 3 - lambda (x): x + y -> lambda x: x + y -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Assign, Name, Newline, Number, Subscript, syms - -def is_docstring(stmt): - return isinstance(stmt, pytree.Node) and \ - stmt.children[0].type == token.STRING - -class FixTupleParams(fixer_base.BaseFix): - run_order = 4 #use a lower order since lambda is part of other - #patterns - BM_compatible = True - - PATTERN = """ - funcdef< 'def' any parameters< '(' args=any ')' > - ['->' any] ':' suite=any+ > - | - lambda= - lambdef< 'lambda' args=vfpdef< '(' inner=any ')' > - ':' body=any - > - """ - - def transform(self, node, results): - if "lambda" in results: - return self.transform_lambda(node, results) - - new_lines = [] - suite = results["suite"] - args = results["args"] - # This crap is so "def foo(...): x = 5; y = 7" is handled correctly. - # TODO(cwinter): suite-cleanup - if suite[0].children[1].type == token.INDENT: - start = 2 - indent = suite[0].children[1].value - end = Newline() - else: - start = 0 - indent = "; " - end = pytree.Leaf(token.INDENT, "") - - # We need access to self for new_name(), and making this a method - # doesn't feel right. Closing over self and new_lines makes the - # code below cleaner. - def handle_tuple(tuple_arg, add_prefix=False): - n = Name(self.new_name()) - arg = tuple_arg.clone() - arg.prefix = "" - stmt = Assign(arg, n.clone()) - if add_prefix: - n.prefix = " " - tuple_arg.replace(n) - new_lines.append(pytree.Node(syms.simple_stmt, - [stmt, end.clone()])) - - if args.type == syms.tfpdef: - handle_tuple(args) - elif args.type == syms.typedargslist: - for i, arg in enumerate(args.children): - if arg.type == syms.tfpdef: - # Without add_prefix, the emitted code is correct, - # just ugly. - handle_tuple(arg, add_prefix=(i > 0)) - - if not new_lines: - return - - # This isn't strictly necessary, but it plays nicely with other fixers. - # TODO(cwinter) get rid of this when children becomes a smart list - for line in new_lines: - line.parent = suite[0] - - # TODO(cwinter) suite-cleanup - after = start - if start == 0: - new_lines[0].prefix = " " - elif is_docstring(suite[0].children[start]): - new_lines[0].prefix = indent - after = start + 1 - - for line in new_lines: - line.parent = suite[0] - suite[0].children[after:after] = new_lines - for i in range(after+1, after+len(new_lines)+1): - suite[0].children[i].prefix = indent - suite[0].changed() - - def transform_lambda(self, node, results): - args = results["args"] - body = results["body"] - inner = simplify_args(results["inner"]) - - # Replace lambda ((((x)))): x with lambda x: x - if inner.type == token.NAME: - inner = inner.clone() - inner.prefix = " " - args.replace(inner) - return - - params = find_params(args) - to_index = map_to_index(params) - tup_name = self.new_name(tuple_name(params)) - - new_param = Name(tup_name, prefix=" ") - args.replace(new_param.clone()) - for n in body.post_order(): - if n.type == token.NAME and n.value in to_index: - subscripts = [c.clone() for c in to_index[n.value]] - new = pytree.Node(syms.power, - [new_param.clone()] + subscripts) - new.prefix = n.prefix - n.replace(new) - - -### Helper functions for transform_lambda() - -def simplify_args(node): - if node.type in (syms.vfplist, token.NAME): - return node - elif node.type == syms.vfpdef: - # These look like vfpdef< '(' x ')' > where x is NAME - # or another vfpdef instance (leading to recursion). - while node.type == syms.vfpdef: - node = node.children[1] - return node - raise RuntimeError("Received unexpected node %s" % node) - -def find_params(node): - if node.type == syms.vfpdef: - return find_params(node.children[1]) - elif node.type == token.NAME: - return node.value - return [find_params(c) for c in node.children if c.type != token.COMMA] - -def map_to_index(param_list, prefix=[], d=None): - if d is None: - d = {} - for i, obj in enumerate(param_list): - trailer = [Subscript(Number(str(i)))] - if isinstance(obj, list): - map_to_index(obj, trailer, d=d) - else: - d[obj] = prefix + trailer - return d - -def tuple_name(param_list): - l = [] - for obj in param_list: - if isinstance(obj, list): - l.append(tuple_name(obj)) - else: - l.append(obj) - return "_".join(l) diff --git a/Lib/lib2to3/fixes/fix_types.py b/Lib/lib2to3/fixes/fix_types.py deleted file mode 100644 index 67bf51f2f5b85a..00000000000000 --- a/Lib/lib2to3/fixes/fix_types.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for removing uses of the types module. - -These work for only the known names in the types module. The forms above -can include types. or not. ie, It is assumed the module is imported either as: - - import types - from types import ... # either * or specific types - -The import statements are not modified. - -There should be another fixer that handles at least the following constants: - - type([]) -> list - type(()) -> tuple - type('') -> str - -""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -_TYPE_MAPPING = { - 'BooleanType' : 'bool', - 'BufferType' : 'memoryview', - 'ClassType' : 'type', - 'ComplexType' : 'complex', - 'DictType': 'dict', - 'DictionaryType' : 'dict', - 'EllipsisType' : 'type(Ellipsis)', - #'FileType' : 'io.IOBase', - 'FloatType': 'float', - 'IntType': 'int', - 'ListType': 'list', - 'LongType': 'int', - 'ObjectType' : 'object', - 'NoneType': 'type(None)', - 'NotImplementedType' : 'type(NotImplemented)', - 'SliceType' : 'slice', - 'StringType': 'bytes', # XXX ? - 'StringTypes' : '(str,)', # XXX ? - 'TupleType': 'tuple', - 'TypeType' : 'type', - 'UnicodeType': 'str', - 'XRangeType' : 'range', - } - -_pats = ["power< 'types' trailer< '.' name='%s' > >" % t for t in _TYPE_MAPPING] - -class FixTypes(fixer_base.BaseFix): - BM_compatible = True - PATTERN = '|'.join(_pats) - - def transform(self, node, results): - new_value = _TYPE_MAPPING.get(results["name"].value) - if new_value: - return Name(new_value, prefix=node.prefix) - return None diff --git a/Lib/lib2to3/fixes/fix_unicode.py b/Lib/lib2to3/fixes/fix_unicode.py deleted file mode 100644 index c7982c2b97c3e1..00000000000000 --- a/Lib/lib2to3/fixes/fix_unicode.py +++ /dev/null @@ -1,42 +0,0 @@ -r"""Fixer for unicode. - -* Changes unicode to str and unichr to chr. - -* If "...\u..." is not unicode literal change it into "...\\u...". - -* Change u"..." into "...". - -""" - -from ..pgen2 import token -from .. import fixer_base - -_mapping = {"unichr" : "chr", "unicode" : "str"} - -class FixUnicode(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "STRING | 'unicode' | 'unichr'" - - def start_tree(self, tree, filename): - super(FixUnicode, self).start_tree(tree, filename) - self.unicode_literals = 'unicode_literals' in tree.future_features - - def transform(self, node, results): - if node.type == token.NAME: - new = node.clone() - new.value = _mapping[node.value] - return new - elif node.type == token.STRING: - val = node.value - if not self.unicode_literals and val[0] in '\'"' and '\\' in val: - val = r'\\'.join([ - v.replace('\\u', r'\\u').replace('\\U', r'\\U') - for v in val.split(r'\\') - ]) - if val[0] in 'uU': - val = val[1:] - if val == node.value: - return node - new = node.clone() - new.value = val - return new diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py deleted file mode 100644 index ab892bc52494c2..00000000000000 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Fix changes imports of urllib which are now incompatible. - This is rather similar to fix_imports, but because of the more - complex nature of the fixing for urllib, it has its own fixer. -""" -# Author: Nick Edds - -# Local imports -from lib2to3.fixes.fix_imports import alternates, FixImports -from lib2to3.fixer_util import (Name, Comma, FromImport, Newline, - find_indentation, Node, syms) - -MAPPING = {"urllib": [ - ("urllib.request", - ["URLopener", "FancyURLopener", "urlretrieve", - "_urlopener", "urlopen", "urlcleanup", - "pathname2url", "url2pathname", "getproxies"]), - ("urllib.parse", - ["quote", "quote_plus", "unquote", "unquote_plus", - "urlencode", "splitattr", "splithost", "splitnport", - "splitpasswd", "splitport", "splitquery", "splittag", - "splittype", "splituser", "splitvalue", ]), - ("urllib.error", - ["ContentTooShortError"])], - "urllib2" : [ - ("urllib.request", - ["urlopen", "install_opener", "build_opener", - "Request", "OpenerDirector", "BaseHandler", - "HTTPDefaultErrorHandler", "HTTPRedirectHandler", - "HTTPCookieProcessor", "ProxyHandler", - "HTTPPasswordMgr", - "HTTPPasswordMgrWithDefaultRealm", - "AbstractBasicAuthHandler", - "HTTPBasicAuthHandler", "ProxyBasicAuthHandler", - "AbstractDigestAuthHandler", - "HTTPDigestAuthHandler", "ProxyDigestAuthHandler", - "HTTPHandler", "HTTPSHandler", "FileHandler", - "FTPHandler", "CacheFTPHandler", - "UnknownHandler"]), - ("urllib.error", - ["URLError", "HTTPError"]), - ] -} - -# Duplicate the url parsing functions for urllib2. -MAPPING["urllib2"].append(MAPPING["urllib"][1]) - - -def build_pattern(): - bare = set() - for old_module, changes in MAPPING.items(): - for change in changes: - new_module, members = change - members = alternates(members) - yield """import_name< 'import' (module=%r - | dotted_as_names< any* module=%r any* >) > - """ % (old_module, old_module) - yield """import_from< 'from' mod_member=%r 'import' - ( member=%s | import_as_name< member=%s 'as' any > | - import_as_names< members=any* >) > - """ % (old_module, members, members) - yield """import_from< 'from' module_star=%r 'import' star='*' > - """ % old_module - yield """import_name< 'import' - dotted_as_name< module_as=%r 'as' any > > - """ % old_module - # bare_with_attr has a special significance for FixImports.match(). - yield """power< bare_with_attr=%r trailer< '.' member=%s > any* > - """ % (old_module, members) - - -class FixUrllib(FixImports): - - def build_pattern(self): - return "|".join(build_pattern()) - - def transform_import(self, node, results): - """Transform for the basic import case. Replaces the old - import name with a comma separated list of its - replacements. - """ - import_mod = results.get("module") - pref = import_mod.prefix - - names = [] - - # create a Node list of the replacement modules - for name in MAPPING[import_mod.value][:-1]: - names.extend([Name(name[0], prefix=pref), Comma()]) - names.append(Name(MAPPING[import_mod.value][-1][0], prefix=pref)) - import_mod.replace(names) - - def transform_member(self, node, results): - """Transform for imports of specific module elements. Replaces - the module to be imported from with the appropriate new - module. - """ - mod_member = results.get("mod_member") - pref = mod_member.prefix - member = results.get("member") - - # Simple case with only a single member being imported - if member: - # this may be a list of length one, or just a node - if isinstance(member, list): - member = member[0] - new_name = None - for change in MAPPING[mod_member.value]: - if member.value in change[1]: - new_name = change[0] - break - if new_name: - mod_member.replace(Name(new_name, prefix=pref)) - else: - self.cannot_convert(node, "This is an invalid module element") - - # Multiple members being imported - else: - # a dictionary for replacements, order matters - modules = [] - mod_dict = {} - members = results["members"] - for member in members: - # we only care about the actual members - if member.type == syms.import_as_name: - as_name = member.children[2].value - member_name = member.children[0].value - else: - member_name = member.value - as_name = None - if member_name != ",": - for change in MAPPING[mod_member.value]: - if member_name in change[1]: - if change[0] not in mod_dict: - modules.append(change[0]) - mod_dict.setdefault(change[0], []).append(member) - - new_nodes = [] - indentation = find_indentation(node) - first = True - def handle_name(name, prefix): - if name.type == syms.import_as_name: - kids = [Name(name.children[0].value, prefix=prefix), - name.children[1].clone(), - name.children[2].clone()] - return [Node(syms.import_as_name, kids)] - return [Name(name.value, prefix=prefix)] - for module in modules: - elts = mod_dict[module] - names = [] - for elt in elts[:-1]: - names.extend(handle_name(elt, pref)) - names.append(Comma()) - names.extend(handle_name(elts[-1], pref)) - new = FromImport(module, names) - if not first or node.parent.prefix.endswith(indentation): - new.prefix = indentation - new_nodes.append(new) - first = False - if new_nodes: - nodes = [] - for new_node in new_nodes[:-1]: - nodes.extend([new_node, Newline()]) - nodes.append(new_nodes[-1]) - node.replace(nodes) - else: - self.cannot_convert(node, "All module elements are invalid") - - def transform_dot(self, node, results): - """Transform for calls to module members in code.""" - module_dot = results.get("bare_with_attr") - member = results.get("member") - new_name = None - if isinstance(member, list): - member = member[0] - for change in MAPPING[module_dot.value]: - if member.value in change[1]: - new_name = change[0] - break - if new_name: - module_dot.replace(Name(new_name, - prefix=module_dot.prefix)) - else: - self.cannot_convert(node, "This is an invalid module element") - - def transform(self, node, results): - if results.get("module"): - self.transform_import(node, results) - elif results.get("mod_member"): - self.transform_member(node, results) - elif results.get("bare_with_attr"): - self.transform_dot(node, results) - # Renaming and star imports are not supported for these modules. - elif results.get("module_star"): - self.cannot_convert(node, "Cannot handle star imports.") - elif results.get("module_as"): - self.cannot_convert(node, "This module is now multiple modules") diff --git a/Lib/lib2to3/fixes/fix_ws_comma.py b/Lib/lib2to3/fixes/fix_ws_comma.py deleted file mode 100644 index a54a376c472afb..00000000000000 --- a/Lib/lib2to3/fixes/fix_ws_comma.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Fixer that changes 'a ,b' into 'a, b'. - -This also changes '{a :b}' into '{a: b}', but does not touch other -uses of colons. It does not touch other uses of whitespace. - -""" - -from .. import pytree -from ..pgen2 import token -from .. import fixer_base - -class FixWsComma(fixer_base.BaseFix): - - explicit = True # The user must ask for this fixers - - PATTERN = """ - any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]> - """ - - COMMA = pytree.Leaf(token.COMMA, ",") - COLON = pytree.Leaf(token.COLON, ":") - SEPS = (COMMA, COLON) - - def transform(self, node, results): - new = node.clone() - comma = False - for child in new.children: - if child in self.SEPS: - prefix = child.prefix - if prefix.isspace() and "\n" not in prefix: - child.prefix = "" - comma = True - else: - if comma: - prefix = child.prefix - if not prefix: - child.prefix = " " - comma = False - return new diff --git a/Lib/lib2to3/fixes/fix_xrange.py b/Lib/lib2to3/fixes/fix_xrange.py deleted file mode 100644 index 1e491e166a3f1c..00000000000000 --- a/Lib/lib2to3/fixes/fix_xrange.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes xrange(...) into range(...).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, Call, consuming_calls -from .. import patcomp - - -class FixXrange(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< - (name='range'|name='xrange') trailer< '(' args=any ')' > - rest=any* > - """ - - def start_tree(self, tree, filename): - super(FixXrange, self).start_tree(tree, filename) - self.transformed_xranges = set() - - def finish_tree(self, tree, filename): - self.transformed_xranges = None - - def transform(self, node, results): - name = results["name"] - if name.value == "xrange": - return self.transform_xrange(node, results) - elif name.value == "range": - return self.transform_range(node, results) - else: - raise ValueError(repr(name)) - - def transform_xrange(self, node, results): - name = results["name"] - name.replace(Name("range", prefix=name.prefix)) - # This prevents the new range call from being wrapped in a list later. - self.transformed_xranges.add(id(node)) - - def transform_range(self, node, results): - if (id(node) not in self.transformed_xranges and - not self.in_special_context(node)): - range_call = Call(Name("range"), [results["args"].clone()]) - # Encase the range call in list(). - list_call = Call(Name("list"), [range_call], - prefix=node.prefix) - # Put things that were after the range() call after the list call. - for n in results["rest"]: - list_call.append_child(n) - return list_call - - P1 = "power< func=NAME trailer< '(' node=any ')' > any* >" - p1 = patcomp.compile_pattern(P1) - - P2 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - | comparison< any 'in' node=any any*> - """ - p2 = patcomp.compile_pattern(P2) - - def in_special_context(self, node): - if node.parent is None: - return False - results = {} - if (node.parent.parent is not None and - self.p1.match(node.parent.parent, results) and - results["node"] is node): - # list(d.keys()) -> list(d.keys()), etc. - return results["func"].value in consuming_calls - # for ... in d.iterkeys() -> for ... in d.keys(), etc. - return self.p2.match(node.parent, results) and results["node"] is node diff --git a/Lib/lib2to3/fixes/fix_xreadlines.py b/Lib/lib2to3/fixes/fix_xreadlines.py deleted file mode 100644 index 3e3f71ab045573..00000000000000 --- a/Lib/lib2to3/fixes/fix_xreadlines.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Fix "for x in f.xreadlines()" -> "for x in f". - -This fixer will also convert g(f.xreadlines) into g(f.__iter__).""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixXreadlines(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< call=any+ trailer< '.' 'xreadlines' > trailer< '(' ')' > > - | - power< any+ trailer< '.' no_call='xreadlines' > > - """ - - def transform(self, node, results): - no_call = results.get("no_call") - - if no_call: - no_call.replace(Name("__iter__", prefix=no_call.prefix)) - else: - node.replace([x.clone() for x in results["call"]]) diff --git a/Lib/lib2to3/fixes/fix_zip.py b/Lib/lib2to3/fixes/fix_zip.py deleted file mode 100644 index 52c28df6aab411..00000000000000 --- a/Lib/lib2to3/fixes/fix_zip.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Fixer that changes zip(seq0, seq1, ...) into list(zip(seq0, seq1, ...) -unless there exists a 'from future_builtins import zip' statement in the -top-level namespace. - -We avoid the transformation if the zip() call is directly contained in -iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or for V in <>:. -""" - -# Local imports -from .. import fixer_base -from ..pytree import Node -from ..pygram import python_symbols as syms -from ..fixer_util import Name, ArgList, in_special_context - - -class FixZip(fixer_base.ConditionalFix): - - BM_compatible = True - PATTERN = """ - power< 'zip' args=trailer< '(' [any] ')' > [trailers=trailer*] - > - """ - - skip_on = "future_builtins.zip" - - def transform(self, node, results): - if self.should_skip(node): - return - - if in_special_context(node): - return None - - args = results['args'].clone() - args.prefix = "" - - trailers = [] - if 'trailers' in results: - trailers = [n.clone() for n in results['trailers']] - for n in trailers: - n.prefix = "" - - new = Node(syms.power, [Name("zip"), args], prefix="") - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py deleted file mode 100644 index f2849fd6be3fd9..00000000000000 --- a/Lib/lib2to3/main.py +++ /dev/null @@ -1,273 +0,0 @@ -""" -Main program for 2to3. -""" - -from __future__ import with_statement, print_function - -import sys -import os -import difflib -import logging -import shutil -import optparse - -from . import refactor - - -def diff_texts(a, b, filename): - """Return a unified diff of two strings.""" - a = a.splitlines() - b = b.splitlines() - return difflib.unified_diff(a, b, filename, filename, - "(original)", "(refactored)", - lineterm="") - - -class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): - """ - A refactoring tool that can avoid overwriting its input files. - Prints output to stdout. - - Output files can optionally be written to a different directory and or - have an extra file suffix appended to their name for use in situations - where you do not want to replace the input files. - """ - - def __init__(self, fixers, options, explicit, nobackups, show_diffs, - input_base_dir='', output_dir='', append_suffix=''): - """ - Args: - fixers: A list of fixers to import. - options: A dict with RefactoringTool configuration. - explicit: A list of fixers to run even if they are explicit. - nobackups: If true no backup '.bak' files will be created for those - files that are being refactored. - show_diffs: Should diffs of the refactoring be printed to stdout? - input_base_dir: The base directory for all input files. This class - will strip this path prefix off of filenames before substituting - it with output_dir. Only meaningful if output_dir is supplied. - All files processed by refactor() must start with this path. - output_dir: If supplied, all converted files will be written into - this directory tree instead of input_base_dir. - append_suffix: If supplied, all files output by this tool will have - this appended to their filename. Useful for changing .py to - .py3 for example by passing append_suffix='3'. - """ - self.nobackups = nobackups - self.show_diffs = show_diffs - if input_base_dir and not input_base_dir.endswith(os.sep): - input_base_dir += os.sep - self._input_base_dir = input_base_dir - self._output_dir = output_dir - self._append_suffix = append_suffix - super(StdoutRefactoringTool, self).__init__(fixers, options, explicit) - - def log_error(self, msg, *args, **kwargs): - self.errors.append((msg, args, kwargs)) - self.logger.error(msg, *args, **kwargs) - - def write_file(self, new_text, filename, old_text, encoding): - orig_filename = filename - if self._output_dir: - if filename.startswith(self._input_base_dir): - filename = os.path.join(self._output_dir, - filename[len(self._input_base_dir):]) - else: - raise ValueError('filename %s does not start with the ' - 'input_base_dir %s' % ( - filename, self._input_base_dir)) - if self._append_suffix: - filename += self._append_suffix - if orig_filename != filename: - output_dir = os.path.dirname(filename) - if not os.path.isdir(output_dir) and output_dir: - os.makedirs(output_dir) - self.log_message('Writing converted %s to %s.', orig_filename, - filename) - if not self.nobackups: - # Make backup - backup = filename + ".bak" - if os.path.lexists(backup): - try: - os.remove(backup) - except OSError: - self.log_message("Can't remove backup %s", backup) - try: - os.rename(filename, backup) - except OSError: - self.log_message("Can't rename %s to %s", filename, backup) - # Actually write the new file - write = super(StdoutRefactoringTool, self).write_file - write(new_text, filename, old_text, encoding) - if not self.nobackups: - shutil.copymode(backup, filename) - if orig_filename != filename: - # Preserve the file mode in the new output directory. - shutil.copymode(orig_filename, filename) - - def print_output(self, old, new, filename, equal): - if equal: - self.log_message("No changes to %s", filename) - else: - self.log_message("Refactored %s", filename) - if self.show_diffs: - diff_lines = diff_texts(old, new, filename) - try: - if self.output_lock is not None: - with self.output_lock: - for line in diff_lines: - print(line) - sys.stdout.flush() - else: - for line in diff_lines: - print(line) - except UnicodeEncodeError: - warn("couldn't encode %s's diff for your terminal" % - (filename,)) - return - -def warn(msg): - print("WARNING: %s" % (msg,), file=sys.stderr) - - -def main(fixer_pkg, args=None): - """Main program. - - Args: - fixer_pkg: the name of a package where the fixers are located. - args: optional; a list of command line arguments. If omitted, - sys.argv[1:] is used. - - Returns a suggested exit status (0, 1, 2). - """ - # Set up option parser - parser = optparse.OptionParser(usage="2to3 [options] file|dir ...") - parser.add_option("-d", "--doctests_only", action="store_true", - help="Fix up doctests only") - parser.add_option("-f", "--fix", action="append", default=[], - help="Each FIX specifies a transformation; default: all") - parser.add_option("-j", "--processes", action="store", default=1, - type="int", help="Run 2to3 concurrently") - parser.add_option("-x", "--nofix", action="append", default=[], - help="Prevent a transformation from being run") - parser.add_option("-l", "--list-fixes", action="store_true", - help="List available transformations") - parser.add_option("-p", "--print-function", action="store_true", - help="Modify the grammar so that print() is a function") - parser.add_option("-e", "--exec-function", action="store_true", - help="Modify the grammar so that exec() is a function") - parser.add_option("-v", "--verbose", action="store_true", - help="More verbose logging") - parser.add_option("--no-diffs", action="store_true", - help="Don't show diffs of the refactoring") - parser.add_option("-w", "--write", action="store_true", - help="Write back modified files") - parser.add_option("-n", "--nobackups", action="store_true", default=False, - help="Don't write backups for modified files") - parser.add_option("-o", "--output-dir", action="store", type="str", - default="", help="Put output files in this directory " - "instead of overwriting the input files. Requires -n.") - parser.add_option("-W", "--write-unchanged-files", action="store_true", - help="Also write files even if no changes were required" - " (useful with --output-dir); implies -w.") - parser.add_option("--add-suffix", action="store", type="str", default="", - help="Append this string to all output filenames." - " Requires -n if non-empty. " - "ex: --add-suffix='3' will generate .py3 files.") - - # Parse command line arguments - refactor_stdin = False - flags = {} - options, args = parser.parse_args(args) - if options.write_unchanged_files: - flags["write_unchanged_files"] = True - if not options.write: - warn("--write-unchanged-files/-W implies -w.") - options.write = True - # If we allowed these, the original files would be renamed to backup names - # but not replaced. - if options.output_dir and not options.nobackups: - parser.error("Can't use --output-dir/-o without -n.") - if options.add_suffix and not options.nobackups: - parser.error("Can't use --add-suffix without -n.") - - if not options.write and options.no_diffs: - warn("not writing files and not printing diffs; that's not very useful") - if not options.write and options.nobackups: - parser.error("Can't use -n without -w") - if options.list_fixes: - print("Available transformations for the -f/--fix option:") - for fixname in refactor.get_all_fix_names(fixer_pkg): - print(fixname) - if not args: - return 0 - if not args: - print("At least one file or directory argument required.", file=sys.stderr) - print("Use --help to show usage.", file=sys.stderr) - return 2 - if "-" in args: - refactor_stdin = True - if options.write: - print("Can't write to stdin.", file=sys.stderr) - return 2 - if options.print_function: - flags["print_function"] = True - - if options.exec_function: - flags["exec_function"] = True - - # Set up logging handler - level = logging.DEBUG if options.verbose else logging.INFO - logging.basicConfig(format='%(name)s: %(message)s', level=level) - logger = logging.getLogger('lib2to3.main') - - # Initialize the refactoring tool - avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg)) - unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix) - explicit = set() - if options.fix: - all_present = False - for fix in options.fix: - if fix == "all": - all_present = True - else: - explicit.add(fixer_pkg + ".fix_" + fix) - requested = avail_fixes.union(explicit) if all_present else explicit - else: - requested = avail_fixes.union(explicit) - fixer_names = requested.difference(unwanted_fixes) - input_base_dir = os.path.commonprefix(args) - if (input_base_dir and not input_base_dir.endswith(os.sep) - and not os.path.isdir(input_base_dir)): - # One or more similar names were passed, their directory is the base. - # os.path.commonprefix() is ignorant of path elements, this corrects - # for that weird API. - input_base_dir = os.path.dirname(input_base_dir) - if options.output_dir: - input_base_dir = input_base_dir.rstrip(os.sep) - logger.info('Output in %r will mirror the input directory %r layout.', - options.output_dir, input_base_dir) - rt = StdoutRefactoringTool( - sorted(fixer_names), flags, sorted(explicit), - options.nobackups, not options.no_diffs, - input_base_dir=input_base_dir, - output_dir=options.output_dir, - append_suffix=options.add_suffix) - - # Refactor all files and directories passed as arguments - if not rt.errors: - if refactor_stdin: - rt.refactor_stdin() - else: - try: - rt.refactor(args, options.write, options.doctests_only, - options.processes) - except refactor.MultiprocessingUnsupported: - assert options.processes > 1 - print("Sorry, -j isn't supported on this platform.", - file=sys.stderr) - return 1 - rt.summarize() - - # Return error status (0 if rt.errors is zero) - return int(bool(rt.errors)) diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py deleted file mode 100644 index f57f4954b26ce7..00000000000000 --- a/Lib/lib2to3/patcomp.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Pattern compiler. - -The grammar is taken from PatternGrammar.txt. - -The compiler compiles a pattern to a pytree.*Pattern instance. -""" - -__author__ = "Guido van Rossum " - -# Python imports -import io - -# Fairly local imports -from .pgen2 import driver, literals, token, tokenize, parse, grammar - -# Really local imports -from . import pytree -from . import pygram - - -class PatternSyntaxError(Exception): - pass - - -def tokenize_wrapper(input): - """Tokenizes a string suppressing significant whitespace.""" - skip = {token.NEWLINE, token.INDENT, token.DEDENT} - tokens = tokenize.generate_tokens(io.StringIO(input).readline) - for quintuple in tokens: - type, value, start, end, line_text = quintuple - if type not in skip: - yield quintuple - - -class PatternCompiler(object): - - def __init__(self, grammar_file=None): - """Initializer. - - Takes an optional alternative filename for the pattern grammar. - """ - if grammar_file is None: - self.grammar = pygram.pattern_grammar - self.syms = pygram.pattern_symbols - else: - self.grammar = driver.load_grammar(grammar_file) - self.syms = pygram.Symbols(self.grammar) - self.pygrammar = pygram.python_grammar - self.pysyms = pygram.python_symbols - self.driver = driver.Driver(self.grammar, convert=pattern_convert) - - def compile_pattern(self, input, debug=False, with_tree=False): - """Compiles a pattern string to a nested pytree.*Pattern object.""" - tokens = tokenize_wrapper(input) - try: - root = self.driver.parse_tokens(tokens, debug=debug) - except parse.ParseError as e: - raise PatternSyntaxError(str(e)) from None - if with_tree: - return self.compile_node(root), root - else: - return self.compile_node(root) - - def compile_node(self, node): - """Compiles a node, recursively. - - This is one big switch on the node type. - """ - # XXX Optimize certain Wildcard-containing-Wildcard patterns - # that can be merged - if node.type == self.syms.Matcher: - node = node.children[0] # Avoid unneeded recursion - - if node.type == self.syms.Alternatives: - # Skip the odd children since they are just '|' tokens - alts = [self.compile_node(ch) for ch in node.children[::2]] - if len(alts) == 1: - return alts[0] - p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1) - return p.optimize() - - if node.type == self.syms.Alternative: - units = [self.compile_node(ch) for ch in node.children] - if len(units) == 1: - return units[0] - p = pytree.WildcardPattern([units], min=1, max=1) - return p.optimize() - - if node.type == self.syms.NegatedUnit: - pattern = self.compile_basic(node.children[1:]) - p = pytree.NegatedPattern(pattern) - return p.optimize() - - assert node.type == self.syms.Unit - - name = None - nodes = node.children - if len(nodes) >= 3 and nodes[1].type == token.EQUAL: - name = nodes[0].value - nodes = nodes[2:] - repeat = None - if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater: - repeat = nodes[-1] - nodes = nodes[:-1] - - # Now we've reduced it to: STRING | NAME [Details] | (...) | [...] - pattern = self.compile_basic(nodes, repeat) - - if repeat is not None: - assert repeat.type == self.syms.Repeater - children = repeat.children - child = children[0] - if child.type == token.STAR: - min = 0 - max = pytree.HUGE - elif child.type == token.PLUS: - min = 1 - max = pytree.HUGE - elif child.type == token.LBRACE: - assert children[-1].type == token.RBRACE - assert len(children) in (3, 5) - min = max = self.get_int(children[1]) - if len(children) == 5: - max = self.get_int(children[3]) - else: - assert False - if min != 1 or max != 1: - pattern = pattern.optimize() - pattern = pytree.WildcardPattern([[pattern]], min=min, max=max) - - if name is not None: - pattern.name = name - return pattern.optimize() - - def compile_basic(self, nodes, repeat=None): - # Compile STRING | NAME [Details] | (...) | [...] - assert len(nodes) >= 1 - node = nodes[0] - if node.type == token.STRING: - value = str(literals.evalString(node.value)) - return pytree.LeafPattern(_type_of_literal(value), value) - elif node.type == token.NAME: - value = node.value - if value.isupper(): - if value not in TOKEN_MAP: - raise PatternSyntaxError("Invalid token: %r" % value) - if nodes[1:]: - raise PatternSyntaxError("Can't have details for token") - return pytree.LeafPattern(TOKEN_MAP[value]) - else: - if value == "any": - type = None - elif not value.startswith("_"): - type = getattr(self.pysyms, value, None) - if type is None: - raise PatternSyntaxError("Invalid symbol: %r" % value) - if nodes[1:]: # Details present - content = [self.compile_node(nodes[1].children[1])] - else: - content = None - return pytree.NodePattern(type, content) - elif node.value == "(": - return self.compile_node(nodes[1]) - elif node.value == "[": - assert repeat is None - subpattern = self.compile_node(nodes[1]) - return pytree.WildcardPattern([[subpattern]], min=0, max=1) - assert False, node - - def get_int(self, node): - assert node.type == token.NUMBER - return int(node.value) - - -# Map named tokens to the type value for a LeafPattern -TOKEN_MAP = {"NAME": token.NAME, - "STRING": token.STRING, - "NUMBER": token.NUMBER, - "TOKEN": None} - - -def _type_of_literal(value): - if value[0].isalpha(): - return token.NAME - elif value in grammar.opmap: - return grammar.opmap[value] - else: - return None - - -def pattern_convert(grammar, raw_node_info): - """Converts raw node information to a Node or Leaf instance.""" - type, value, context, children = raw_node_info - if children or type in grammar.number2symbol: - return pytree.Node(type, children, context=context) - else: - return pytree.Leaf(type, value, context=context) - - -def compile_pattern(pattern): - return PatternCompiler().compile_pattern(pattern) diff --git a/Lib/lib2to3/pgen2/__init__.py b/Lib/lib2to3/pgen2/__init__.py deleted file mode 100644 index af390484528d87..00000000000000 --- a/Lib/lib2to3/pgen2/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""The pgen2 package.""" diff --git a/Lib/lib2to3/pgen2/conv.py b/Lib/lib2to3/pgen2/conv.py deleted file mode 100644 index ed0cac532e4249..00000000000000 --- a/Lib/lib2to3/pgen2/conv.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Convert graminit.[ch] spit out by pgen to Python code. - -Pgen is the Python parser generator. It is useful to quickly create a -parser from a grammar file in Python's grammar notation. But I don't -want my parsers to be written in C (yet), so I'm translating the -parsing tables to Python data structures and writing a Python parse -engine. - -Note that the token numbers are constants determined by the standard -Python tokenizer. The standard token module defines these numbers and -their names (the names are not used much). The token numbers are -hardcoded into the Python tokenizer and into pgen. A Python -implementation of the Python tokenizer is also available, in the -standard tokenize module. - -On the other hand, symbol numbers (representing the grammar's -non-terminals) are assigned by pgen based on the actual grammar -input. - -Note: this module is pretty much obsolete; the pgen module generates -equivalent grammar tables directly from the Grammar.txt input file -without having to invoke the Python pgen C program. - -""" - -# Python imports -import re - -# Local imports -from pgen2 import grammar, token - - -class Converter(grammar.Grammar): - """Grammar subclass that reads classic pgen output files. - - The run() method reads the tables as produced by the pgen parser - generator, typically contained in two C files, graminit.h and - graminit.c. The other methods are for internal use only. - - See the base class for more documentation. - - """ - - def run(self, graminit_h, graminit_c): - """Load the grammar tables from the text files written by pgen.""" - self.parse_graminit_h(graminit_h) - self.parse_graminit_c(graminit_c) - self.finish_off() - - def parse_graminit_h(self, filename): - """Parse the .h file written by pgen. (Internal) - - This file is a sequence of #define statements defining the - nonterminals of the grammar as numbers. We build two tables - mapping the numbers to names and back. - - """ - try: - f = open(filename) - except OSError as err: - print("Can't open %s: %s" % (filename, err)) - return False - self.symbol2number = {} - self.number2symbol = {} - lineno = 0 - for line in f: - lineno += 1 - mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line) - if not mo and line.strip(): - print("%s(%s): can't parse %s" % (filename, lineno, - line.strip())) - else: - symbol, number = mo.groups() - number = int(number) - assert symbol not in self.symbol2number - assert number not in self.number2symbol - self.symbol2number[symbol] = number - self.number2symbol[number] = symbol - return True - - def parse_graminit_c(self, filename): - """Parse the .c file written by pgen. (Internal) - - The file looks as follows. The first two lines are always this: - - #include "pgenheaders.h" - #include "grammar.h" - - After that come four blocks: - - 1) one or more state definitions - 2) a table defining dfas - 3) a table defining labels - 4) a struct defining the grammar - - A state definition has the following form: - - one or more arc arrays, each of the form: - static arc arcs__[] = { - {, }, - ... - }; - - followed by a state array, of the form: - static state states_[] = { - {, arcs__}, - ... - }; - - """ - try: - f = open(filename) - except OSError as err: - print("Can't open %s: %s" % (filename, err)) - return False - # The code below essentially uses f's iterator-ness! - lineno = 0 - - # Expect the two #include lines - lineno, line = lineno+1, next(f) - assert line == '#include "pgenheaders.h"\n', (lineno, line) - lineno, line = lineno+1, next(f) - assert line == '#include "grammar.h"\n', (lineno, line) - - # Parse the state definitions - lineno, line = lineno+1, next(f) - allarcs = {} - states = [] - while line.startswith("static arc "): - while line.startswith("static arc "): - mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$", - line) - assert mo, (lineno, line) - n, m, k = list(map(int, mo.groups())) - arcs = [] - for _ in range(k): - lineno, line = lineno+1, next(f) - mo = re.match(r"\s+{(\d+), (\d+)},$", line) - assert mo, (lineno, line) - i, j = list(map(int, mo.groups())) - arcs.append((i, j)) - lineno, line = lineno+1, next(f) - assert line == "};\n", (lineno, line) - allarcs[(n, m)] = arcs - lineno, line = lineno+1, next(f) - mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line) - assert mo, (lineno, line) - s, t = list(map(int, mo.groups())) - assert s == len(states), (lineno, line) - state = [] - for _ in range(t): - lineno, line = lineno+1, next(f) - mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line) - assert mo, (lineno, line) - k, n, m = list(map(int, mo.groups())) - arcs = allarcs[n, m] - assert k == len(arcs), (lineno, line) - state.append(arcs) - states.append(state) - lineno, line = lineno+1, next(f) - assert line == "};\n", (lineno, line) - lineno, line = lineno+1, next(f) - self.states = states - - # Parse the dfas - dfas = {} - mo = re.match(r"static dfa dfas\[(\d+)\] = {$", line) - assert mo, (lineno, line) - ndfas = int(mo.group(1)) - for i in range(ndfas): - lineno, line = lineno+1, next(f) - mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', - line) - assert mo, (lineno, line) - symbol = mo.group(2) - number, x, y, z = list(map(int, mo.group(1, 3, 4, 5))) - assert self.symbol2number[symbol] == number, (lineno, line) - assert self.number2symbol[number] == symbol, (lineno, line) - assert x == 0, (lineno, line) - state = states[z] - assert y == len(state), (lineno, line) - lineno, line = lineno+1, next(f) - mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line) - assert mo, (lineno, line) - first = {} - rawbitset = eval(mo.group(1)) - for i, c in enumerate(rawbitset): - byte = ord(c) - for j in range(8): - if byte & (1<= os.path.getmtime(b) - - -def load_packaged_grammar(package, grammar_source): - """Normally, loads a pickled grammar by doing - pkgutil.get_data(package, pickled_grammar) - where *pickled_grammar* is computed from *grammar_source* by adding the - Python version and using a ``.pickle`` extension. - - However, if *grammar_source* is an extant file, load_grammar(grammar_source) - is called instead. This facilitates using a packaged grammar file when needed - but preserves load_grammar's automatic regeneration behavior when possible. - - """ - if os.path.isfile(grammar_source): - return load_grammar(grammar_source) - pickled_name = _generate_pickle_name(os.path.basename(grammar_source)) - data = pkgutil.get_data(package, pickled_name) - g = grammar.Grammar() - g.loads(data) - return g - - -def main(*args): - """Main program, when run as a script: produce grammar pickle files. - - Calls load_grammar for each argument, a path to a grammar text file. - """ - if not args: - args = sys.argv[1:] - logging.basicConfig(level=logging.INFO, stream=sys.stdout, - format='%(message)s') - for gt in args: - load_grammar(gt, save=True, force=True) - return True - -if __name__ == "__main__": - sys.exit(int(not main())) diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py deleted file mode 100644 index 5d550aeb65e8dd..00000000000000 --- a/Lib/lib2to3/pgen2/grammar.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""This module defines the data structures used to represent a grammar. - -These are a bit arcane because they are derived from the data -structures used by Python's 'pgen' parser generator. - -There's also a table here mapping operators to their names in the -token module; the Python tokenize module reports all operators as the -fallback token code OP, but the parser needs the actual token code. - -""" - -# Python imports -import pickle - -# Local imports -from . import token - - -class Grammar(object): - """Pgen parsing tables conversion class. - - Once initialized, this class supplies the grammar tables for the - parsing engine implemented by parse.py. The parsing engine - accesses the instance variables directly. The class here does not - provide initialization of the tables; several subclasses exist to - do this (see the conv and pgen modules). - - The load() method reads the tables from a pickle file, which is - much faster than the other ways offered by subclasses. The pickle - file is written by calling dump() (after loading the grammar - tables using a subclass). The report() method prints a readable - representation of the tables to stdout, for debugging. - - The instance variables are as follows: - - symbol2number -- a dict mapping symbol names to numbers. Symbol - numbers are always 256 or higher, to distinguish - them from token numbers, which are between 0 and - 255 (inclusive). - - number2symbol -- a dict mapping numbers to symbol names; - these two are each other's inverse. - - states -- a list of DFAs, where each DFA is a list of - states, each state is a list of arcs, and each - arc is a (i, j) pair where i is a label and j is - a state number. The DFA number is the index into - this list. (This name is slightly confusing.) - Final states are represented by a special arc of - the form (0, j) where j is its own state number. - - dfas -- a dict mapping symbol numbers to (DFA, first) - pairs, where DFA is an item from the states list - above, and first is a set of tokens that can - begin this grammar rule (represented by a dict - whose values are always 1). - - labels -- a list of (x, y) pairs where x is either a token - number or a symbol number, and y is either None - or a string; the strings are keywords. The label - number is the index in this list; label numbers - are used to mark state transitions (arcs) in the - DFAs. - - start -- the number of the grammar's start symbol. - - keywords -- a dict mapping keyword strings to arc labels. - - tokens -- a dict mapping token numbers to arc labels. - - """ - - def __init__(self): - self.symbol2number = {} - self.number2symbol = {} - self.states = [] - self.dfas = {} - self.labels = [(0, "EMPTY")] - self.keywords = {} - self.tokens = {} - self.symbol2label = {} - self.start = 256 - - def dump(self, filename): - """Dump the grammar tables to a pickle file.""" - with open(filename, "wb") as f: - pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL) - - def load(self, filename): - """Load the grammar tables from a pickle file.""" - with open(filename, "rb") as f: - d = pickle.load(f) - self.__dict__.update(d) - - def loads(self, pkl): - """Load the grammar tables from a pickle bytes object.""" - self.__dict__.update(pickle.loads(pkl)) - - def copy(self): - """ - Copy the grammar. - """ - new = self.__class__() - for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords", - "tokens", "symbol2label"): - setattr(new, dict_attr, getattr(self, dict_attr).copy()) - new.labels = self.labels[:] - new.states = self.states[:] - new.start = self.start - return new - - def report(self): - """Dump the grammar tables to standard output, for debugging.""" - from pprint import pprint - print("s2n") - pprint(self.symbol2number) - print("n2s") - pprint(self.number2symbol) - print("states") - pprint(self.states) - print("dfas") - pprint(self.dfas) - print("labels") - pprint(self.labels) - print("start", self.start) - - -# Map from operator to number (since tokenize doesn't do this) - -opmap_raw = """ -( LPAR -) RPAR -[ LSQB -] RSQB -: COLON -, COMMA -; SEMI -+ PLUS -- MINUS -* STAR -/ SLASH -| VBAR -& AMPER -< LESS -> GREATER -= EQUAL -. DOT -% PERCENT -` BACKQUOTE -{ LBRACE -} RBRACE -@ AT -@= ATEQUAL -== EQEQUAL -!= NOTEQUAL -<> NOTEQUAL -<= LESSEQUAL ->= GREATEREQUAL -~ TILDE -^ CIRCUMFLEX -<< LEFTSHIFT ->> RIGHTSHIFT -** DOUBLESTAR -+= PLUSEQUAL --= MINEQUAL -*= STAREQUAL -/= SLASHEQUAL -%= PERCENTEQUAL -&= AMPEREQUAL -|= VBAREQUAL -^= CIRCUMFLEXEQUAL -<<= LEFTSHIFTEQUAL ->>= RIGHTSHIFTEQUAL -**= DOUBLESTAREQUAL -// DOUBLESLASH -//= DOUBLESLASHEQUAL --> RARROW -:= COLONEQUAL -""" - -opmap = {} -for line in opmap_raw.splitlines(): - if line: - op, name = line.split() - opmap[op] = getattr(token, name) -del line, op, name diff --git a/Lib/lib2to3/pgen2/literals.py b/Lib/lib2to3/pgen2/literals.py deleted file mode 100644 index b9b63e6e5572c1..00000000000000 --- a/Lib/lib2to3/pgen2/literals.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Safely evaluate Python string literals without using eval().""" - -import re - -simple_escapes = {"a": "\a", - "b": "\b", - "f": "\f", - "n": "\n", - "r": "\r", - "t": "\t", - "v": "\v", - "'": "'", - '"': '"', - "\\": "\\"} - -def escape(m): - all, tail = m.group(0, 1) - assert all.startswith("\\") - esc = simple_escapes.get(tail) - if esc is not None: - return esc - if tail.startswith("x"): - hexes = tail[1:] - if len(hexes) < 2: - raise ValueError("invalid hex string escape ('\\%s')" % tail) - try: - i = int(hexes, 16) - except ValueError: - raise ValueError("invalid hex string escape ('\\%s')" % tail) from None - else: - try: - i = int(tail, 8) - except ValueError: - raise ValueError("invalid octal string escape ('\\%s')" % tail) from None - return chr(i) - -def evalString(s): - assert s.startswith("'") or s.startswith('"'), repr(s[:1]) - q = s[0] - if s[:3] == q*3: - q = q*3 - assert s.endswith(q), repr(s[-len(q):]) - assert len(s) >= 2*len(q) - s = s[len(q):-len(q)] - return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s) - -def test(): - for i in range(256): - c = chr(i) - s = repr(c) - e = evalString(s) - if e != c: - print(i, c, s, e) - - -if __name__ == "__main__": - test() diff --git a/Lib/lib2to3/pgen2/parse.py b/Lib/lib2to3/pgen2/parse.py deleted file mode 100644 index cf3fcf7e99fd11..00000000000000 --- a/Lib/lib2to3/pgen2/parse.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Parser engine for the grammar tables generated by pgen. - -The grammar table must be loaded first. - -See Parser/parser.c in the Python distribution for additional info on -how this parsing engine works. - -""" - -# Local imports -from . import token - -class ParseError(Exception): - """Exception to signal the parser is stuck.""" - - def __init__(self, msg, type, value, context): - Exception.__init__(self, "%s: type=%r, value=%r, context=%r" % - (msg, type, value, context)) - self.msg = msg - self.type = type - self.value = value - self.context = context - - def __reduce__(self): - return type(self), (self.msg, self.type, self.value, self.context) - -class Parser(object): - """Parser engine. - - The proper usage sequence is: - - p = Parser(grammar, [converter]) # create instance - p.setup([start]) # prepare for parsing - : - if p.addtoken(...): # parse a token; may raise ParseError - break - root = p.rootnode # root of abstract syntax tree - - A Parser instance may be reused by calling setup() repeatedly. - - A Parser instance contains state pertaining to the current token - sequence, and should not be used concurrently by different threads - to parse separate token sequences. - - See driver.py for how to get input tokens by tokenizing a file or - string. - - Parsing is complete when addtoken() returns True; the root of the - abstract syntax tree can then be retrieved from the rootnode - instance variable. When a syntax error occurs, addtoken() raises - the ParseError exception. There is no error recovery; the parser - cannot be used after a syntax error was reported (but it can be - reinitialized by calling setup()). - - """ - - def __init__(self, grammar, convert=None): - """Constructor. - - The grammar argument is a grammar.Grammar instance; see the - grammar module for more information. - - The parser is not ready yet for parsing; you must call the - setup() method to get it started. - - The optional convert argument is a function mapping concrete - syntax tree nodes to abstract syntax tree nodes. If not - given, no conversion is done and the syntax tree produced is - the concrete syntax tree. If given, it must be a function of - two arguments, the first being the grammar (a grammar.Grammar - instance), and the second being the concrete syntax tree node - to be converted. The syntax tree is converted from the bottom - up. - - A concrete syntax tree node is a (type, value, context, nodes) - tuple, where type is the node type (a token or symbol number), - value is None for symbols and a string for tokens, context is - None or an opaque value used for error reporting (typically a - (lineno, offset) pair), and nodes is a list of children for - symbols, and None for tokens. - - An abstract syntax tree node may be anything; this is entirely - up to the converter function. - - """ - self.grammar = grammar - self.convert = convert or (lambda grammar, node: node) - - def setup(self, start=None): - """Prepare for parsing. - - This *must* be called before starting to parse. - - The optional argument is an alternative start symbol; it - defaults to the grammar's start symbol. - - You can use a Parser instance to parse any number of programs; - each time you call setup() the parser is reset to an initial - state determined by the (implicit or explicit) start symbol. - - """ - if start is None: - start = self.grammar.start - # Each stack entry is a tuple: (dfa, state, node). - # A node is a tuple: (type, value, context, children), - # where children is a list of nodes or None, and context may be None. - newnode = (start, None, None, []) - stackentry = (self.grammar.dfas[start], 0, newnode) - self.stack = [stackentry] - self.rootnode = None - self.used_names = set() # Aliased to self.rootnode.used_names in pop() - - def addtoken(self, type, value, context): - """Add a token; return True iff this is the end of the program.""" - # Map from token to label - ilabel = self.classify(type, value, context) - # Loop until the token is shifted; may raise exceptions - while True: - dfa, state, node = self.stack[-1] - states, first = dfa - arcs = states[state] - # Look for a state with this label - for i, newstate in arcs: - t, v = self.grammar.labels[i] - if ilabel == i: - # Look it up in the list of labels - assert t < 256 - # Shift a token; we're done with it - self.shift(type, value, newstate, context) - # Pop while we are in an accept-only state - state = newstate - while states[state] == [(0, state)]: - self.pop() - if not self.stack: - # Done parsing! - return True - dfa, state, node = self.stack[-1] - states, first = dfa - # Done with this token - return False - elif t >= 256: - # See if it's a symbol and if we're in its first set - itsdfa = self.grammar.dfas[t] - itsstates, itsfirst = itsdfa - if ilabel in itsfirst: - # Push a symbol - self.push(t, self.grammar.dfas[t], newstate, context) - break # To continue the outer while loop - else: - if (0, state) in arcs: - # An accepting state, pop it and try something else - self.pop() - if not self.stack: - # Done parsing, but another token is input - raise ParseError("too much input", - type, value, context) - else: - # No success finding a transition - raise ParseError("bad input", type, value, context) - - def classify(self, type, value, context): - """Turn a token into a label. (Internal)""" - if type == token.NAME: - # Keep a listing of all used names - self.used_names.add(value) - # Check for reserved words - ilabel = self.grammar.keywords.get(value) - if ilabel is not None: - return ilabel - ilabel = self.grammar.tokens.get(type) - if ilabel is None: - raise ParseError("bad token", type, value, context) - return ilabel - - def shift(self, type, value, newstate, context): - """Shift a token. (Internal)""" - dfa, state, node = self.stack[-1] - newnode = (type, value, context, None) - newnode = self.convert(self.grammar, newnode) - if newnode is not None: - node[-1].append(newnode) - self.stack[-1] = (dfa, newstate, node) - - def push(self, type, newdfa, newstate, context): - """Push a nonterminal. (Internal)""" - dfa, state, node = self.stack[-1] - newnode = (type, None, context, []) - self.stack[-1] = (dfa, newstate, node) - self.stack.append((newdfa, 0, newnode)) - - def pop(self): - """Pop a nonterminal. (Internal)""" - popdfa, popstate, popnode = self.stack.pop() - newnode = self.convert(self.grammar, popnode) - if newnode is not None: - if self.stack: - dfa, state, node = self.stack[-1] - node[-1].append(newnode) - else: - self.rootnode = newnode - self.rootnode.used_names = self.used_names diff --git a/Lib/lib2to3/pgen2/pgen.py b/Lib/lib2to3/pgen2/pgen.py deleted file mode 100644 index 7abd5cef1c36bb..00000000000000 --- a/Lib/lib2to3/pgen2/pgen.py +++ /dev/null @@ -1,386 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Pgen imports -from . import grammar, token, tokenize - -class PgenGrammar(grammar.Grammar): - pass - -class ParserGenerator(object): - - def __init__(self, filename, stream=None): - close_stream = None - if stream is None: - stream = open(filename, encoding="utf-8") - close_stream = stream.close - self.filename = filename - self.stream = stream - self.generator = tokenize.generate_tokens(stream.readline) - self.gettoken() # Initialize lookahead - self.dfas, self.startsymbol = self.parse() - if close_stream is not None: - close_stream() - self.first = {} # map from symbol name to set of tokens - self.addfirstsets() - - def make_grammar(self): - c = PgenGrammar() - names = list(self.dfas.keys()) - names.sort() - names.remove(self.startsymbol) - names.insert(0, self.startsymbol) - for name in names: - i = 256 + len(c.symbol2number) - c.symbol2number[name] = i - c.number2symbol[i] = name - for name in names: - dfa = self.dfas[name] - states = [] - for state in dfa: - arcs = [] - for label, next in sorted(state.arcs.items()): - arcs.append((self.make_label(c, label), dfa.index(next))) - if state.isfinal: - arcs.append((0, dfa.index(state))) - states.append(arcs) - c.states.append(states) - c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) - c.start = c.symbol2number[self.startsymbol] - return c - - def make_first(self, c, name): - rawfirst = self.first[name] - first = {} - for label in sorted(rawfirst): - ilabel = self.make_label(c, label) - ##assert ilabel not in first # XXX failed on <> ... != - first[ilabel] = 1 - return first - - def make_label(self, c, label): - # XXX Maybe this should be a method on a subclass of converter? - ilabel = len(c.labels) - if label[0].isalpha(): - # Either a symbol name or a named token - if label in c.symbol2number: - # A symbol name (a non-terminal) - if label in c.symbol2label: - return c.symbol2label[label] - else: - c.labels.append((c.symbol2number[label], None)) - c.symbol2label[label] = ilabel - return ilabel - else: - # A named token (NAME, NUMBER, STRING) - itoken = getattr(token, label, None) - assert isinstance(itoken, int), label - assert itoken in token.tok_name, label - if itoken in c.tokens: - return c.tokens[itoken] - else: - c.labels.append((itoken, None)) - c.tokens[itoken] = ilabel - return ilabel - else: - # Either a keyword or an operator - assert label[0] in ('"', "'"), label - value = eval(label) - if value[0].isalpha(): - # A keyword - if value in c.keywords: - return c.keywords[value] - else: - c.labels.append((token.NAME, value)) - c.keywords[value] = ilabel - return ilabel - else: - # An operator (any non-numeric token) - itoken = grammar.opmap[value] # Fails if unknown token - if itoken in c.tokens: - return c.tokens[itoken] - else: - c.labels.append((itoken, None)) - c.tokens[itoken] = ilabel - return ilabel - - def addfirstsets(self): - names = list(self.dfas.keys()) - names.sort() - for name in names: - if name not in self.first: - self.calcfirst(name) - #print name, self.first[name].keys() - - def calcfirst(self, name): - dfa = self.dfas[name] - self.first[name] = None # dummy to detect left recursion - state = dfa[0] - totalset = {} - overlapcheck = {} - for label, next in state.arcs.items(): - if label in self.dfas: - if label in self.first: - fset = self.first[label] - if fset is None: - raise ValueError("recursion for rule %r" % name) - else: - self.calcfirst(label) - fset = self.first[label] - totalset.update(fset) - overlapcheck[label] = fset - else: - totalset[label] = 1 - overlapcheck[label] = {label: 1} - inverse = {} - for label, itsfirst in overlapcheck.items(): - for symbol in itsfirst: - if symbol in inverse: - raise ValueError("rule %s is ambiguous; %s is in the" - " first sets of %s as well as %s" % - (name, symbol, label, inverse[symbol])) - inverse[symbol] = label - self.first[name] = totalset - - def parse(self): - dfas = {} - startsymbol = None - # MSTART: (NEWLINE | RULE)* ENDMARKER - while self.type != token.ENDMARKER: - while self.type == token.NEWLINE: - self.gettoken() - # RULE: NAME ':' RHS NEWLINE - name = self.expect(token.NAME) - self.expect(token.OP, ":") - a, z = self.parse_rhs() - self.expect(token.NEWLINE) - #self.dump_nfa(name, a, z) - dfa = self.make_dfa(a, z) - #self.dump_dfa(name, dfa) - oldlen = len(dfa) - self.simplify_dfa(dfa) - newlen = len(dfa) - dfas[name] = dfa - #print name, oldlen, newlen - if startsymbol is None: - startsymbol = name - return dfas, startsymbol - - def make_dfa(self, start, finish): - # To turn an NFA into a DFA, we define the states of the DFA - # to correspond to *sets* of states of the NFA. Then do some - # state reduction. Let's represent sets as dicts with 1 for - # values. - assert isinstance(start, NFAState) - assert isinstance(finish, NFAState) - def closure(state): - base = {} - addclosure(state, base) - return base - def addclosure(state, base): - assert isinstance(state, NFAState) - if state in base: - return - base[state] = 1 - for label, next in state.arcs: - if label is None: - addclosure(next, base) - states = [DFAState(closure(start), finish)] - for state in states: # NB states grows while we're iterating - arcs = {} - for nfastate in state.nfaset: - for label, next in nfastate.arcs: - if label is not None: - addclosure(next, arcs.setdefault(label, {})) - for label, nfaset in sorted(arcs.items()): - for st in states: - if st.nfaset == nfaset: - break - else: - st = DFAState(nfaset, finish) - states.append(st) - state.addarc(st, label) - return states # List of DFAState instances; first one is start - - def dump_nfa(self, name, start, finish): - print("Dump of NFA for", name) - todo = [start] - for i, state in enumerate(todo): - print(" State", i, state is finish and "(final)" or "") - for label, next in state.arcs: - if next in todo: - j = todo.index(next) - else: - j = len(todo) - todo.append(next) - if label is None: - print(" -> %d" % j) - else: - print(" %s -> %d" % (label, j)) - - def dump_dfa(self, name, dfa): - print("Dump of DFA for", name) - for i, state in enumerate(dfa): - print(" State", i, state.isfinal and "(final)" or "") - for label, next in sorted(state.arcs.items()): - print(" %s -> %d" % (label, dfa.index(next))) - - def simplify_dfa(self, dfa): - # This is not theoretically optimal, but works well enough. - # Algorithm: repeatedly look for two states that have the same - # set of arcs (same labels pointing to the same nodes) and - # unify them, until things stop changing. - - # dfa is a list of DFAState instances - changes = True - while changes: - changes = False - for i, state_i in enumerate(dfa): - for j in range(i+1, len(dfa)): - state_j = dfa[j] - if state_i == state_j: - #print " unify", i, j - del dfa[j] - for state in dfa: - state.unifystate(state_j, state_i) - changes = True - break - - def parse_rhs(self): - # RHS: ALT ('|' ALT)* - a, z = self.parse_alt() - if self.value != "|": - return a, z - else: - aa = NFAState() - zz = NFAState() - aa.addarc(a) - z.addarc(zz) - while self.value == "|": - self.gettoken() - a, z = self.parse_alt() - aa.addarc(a) - z.addarc(zz) - return aa, zz - - def parse_alt(self): - # ALT: ITEM+ - a, b = self.parse_item() - while (self.value in ("(", "[") or - self.type in (token.NAME, token.STRING)): - c, d = self.parse_item() - b.addarc(c) - b = d - return a, b - - def parse_item(self): - # ITEM: '[' RHS ']' | ATOM ['+' | '*'] - if self.value == "[": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, "]") - a.addarc(z) - return a, z - else: - a, z = self.parse_atom() - value = self.value - if value not in ("+", "*"): - return a, z - self.gettoken() - z.addarc(a) - if value == "+": - return a, z - else: - return a, a - - def parse_atom(self): - # ATOM: '(' RHS ')' | NAME | STRING - if self.value == "(": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, ")") - return a, z - elif self.type in (token.NAME, token.STRING): - a = NFAState() - z = NFAState() - a.addarc(z, self.value) - self.gettoken() - return a, z - else: - self.raise_error("expected (...) or NAME or STRING, got %s/%s", - self.type, self.value) - - def expect(self, type, value=None): - if self.type != type or (value is not None and self.value != value): - self.raise_error("expected %s/%s, got %s/%s", - type, value, self.type, self.value) - value = self.value - self.gettoken() - return value - - def gettoken(self): - tup = next(self.generator) - while tup[0] in (tokenize.COMMENT, tokenize.NL): - tup = next(self.generator) - self.type, self.value, self.begin, self.end, self.line = tup - #print token.tok_name[self.type], repr(self.value) - - def raise_error(self, msg, *args): - if args: - try: - msg = msg % args - except: - msg = " ".join([msg] + list(map(str, args))) - raise SyntaxError(msg, (self.filename, self.end[0], - self.end[1], self.line)) - -class NFAState(object): - - def __init__(self): - self.arcs = [] # list of (label, NFAState) pairs - - def addarc(self, next, label=None): - assert label is None or isinstance(label, str) - assert isinstance(next, NFAState) - self.arcs.append((label, next)) - -class DFAState(object): - - def __init__(self, nfaset, final): - assert isinstance(nfaset, dict) - assert isinstance(next(iter(nfaset)), NFAState) - assert isinstance(final, NFAState) - self.nfaset = nfaset - self.isfinal = final in nfaset - self.arcs = {} # map from label to DFAState - - def addarc(self, next, label): - assert isinstance(label, str) - assert label not in self.arcs - assert isinstance(next, DFAState) - self.arcs[label] = next - - def unifystate(self, old, new): - for label, next in self.arcs.items(): - if next is old: - self.arcs[label] = new - - def __eq__(self, other): - # Equality test -- ignore the nfaset instance variable - assert isinstance(other, DFAState) - if self.isfinal != other.isfinal: - return False - # Can't just return self.arcs == other.arcs, because that - # would invoke this method recursively, with cycles... - if len(self.arcs) != len(other.arcs): - return False - for label, next in self.arcs.items(): - if next is not other.arcs.get(label): - return False - return True - - __hash__ = None # For Py3 compatibility. - -def generate_grammar(filename="Grammar.txt"): - p = ParserGenerator(filename) - return p.make_grammar() diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py deleted file mode 100755 index 2a55138e48237b..00000000000000 --- a/Lib/lib2to3/pgen2/token.py +++ /dev/null @@ -1,86 +0,0 @@ -#! /usr/bin/env python3 - -"""Token constants (from "token.h").""" - -# Taken from Python (r53757) and modified to include some tokens -# originally monkeypatched in by pgen2.tokenize - -#--start constants-- -ENDMARKER = 0 -NAME = 1 -NUMBER = 2 -STRING = 3 -NEWLINE = 4 -INDENT = 5 -DEDENT = 6 -LPAR = 7 -RPAR = 8 -LSQB = 9 -RSQB = 10 -COLON = 11 -COMMA = 12 -SEMI = 13 -PLUS = 14 -MINUS = 15 -STAR = 16 -SLASH = 17 -VBAR = 18 -AMPER = 19 -LESS = 20 -GREATER = 21 -EQUAL = 22 -DOT = 23 -PERCENT = 24 -BACKQUOTE = 25 -LBRACE = 26 -RBRACE = 27 -EQEQUAL = 28 -NOTEQUAL = 29 -LESSEQUAL = 30 -GREATEREQUAL = 31 -TILDE = 32 -CIRCUMFLEX = 33 -LEFTSHIFT = 34 -RIGHTSHIFT = 35 -DOUBLESTAR = 36 -PLUSEQUAL = 37 -MINEQUAL = 38 -STAREQUAL = 39 -SLASHEQUAL = 40 -PERCENTEQUAL = 41 -AMPEREQUAL = 42 -VBAREQUAL = 43 -CIRCUMFLEXEQUAL = 44 -LEFTSHIFTEQUAL = 45 -RIGHTSHIFTEQUAL = 46 -DOUBLESTAREQUAL = 47 -DOUBLESLASH = 48 -DOUBLESLASHEQUAL = 49 -AT = 50 -ATEQUAL = 51 -OP = 52 -COMMENT = 53 -NL = 54 -RARROW = 55 -AWAIT = 56 -ASYNC = 57 -ERRORTOKEN = 58 -COLONEQUAL = 59 -N_TOKENS = 60 -NT_OFFSET = 256 -#--end constants-- - -tok_name = {} -for _name, _value in list(globals().items()): - if isinstance(_value, int): - tok_name[_value] = _name - - -def ISTERMINAL(x): - return x < NT_OFFSET - -def ISNONTERMINAL(x): - return x >= NT_OFFSET - -def ISEOF(x): - return x == ENDMARKER diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py deleted file mode 100644 index 099dfa7798afd4..00000000000000 --- a/Lib/lib2to3/pgen2/tokenize.py +++ /dev/null @@ -1,564 +0,0 @@ -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation. -# All rights reserved. - -"""Tokenization help for Python programs. - -generate_tokens(readline) is a generator that breaks a stream of -text into Python tokens. It accepts a readline-like method which is called -repeatedly to get the next line of input (or "" for EOF). It generates -5-tuples with these members: - - the token type (see token.py) - the token (a string) - the starting (row, column) indices of the token (a 2-tuple of ints) - the ending (row, column) indices of the token (a 2-tuple of ints) - the original line (string) - -It is designed to match the working of the Python tokenizer exactly, except -that it produces COMMENT tokens for comments and gives type OP for all -operators - -Older entry points - tokenize_loop(readline, tokeneater) - tokenize(readline, tokeneater=printtoken) -are the same, except instead of generating tokens, tokeneater is a callback -function to which the 5 fields described above are passed as 5 arguments, -each time a new token is found.""" - -__author__ = 'Ka-Ping Yee ' -__credits__ = \ - 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro' - -import string, re -from codecs import BOM_UTF8, lookup -from lib2to3.pgen2.token import * - -from . import token -__all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize", - "generate_tokens", "untokenize"] -del token - -try: - bytes -except NameError: - # Support bytes type in Python <= 2.5, so 2to3 turns itself into - # valid Python 3 code. - bytes = str - -def group(*choices): return '(' + '|'.join(choices) + ')' -def any(*choices): return group(*choices) + '*' -def maybe(*choices): return group(*choices) + '?' -def _combinations(*l): - return set( - x + y for x in l for y in l + ("",) if x.casefold() != y.casefold() - ) - -Whitespace = r'[ \f\t]*' -Comment = r'#[^\r\n]*' -Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) -Name = r'\w+' - -Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' -Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' -Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' -Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') -Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+(?:_\d+)*' -Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) -Expfloat = r'\d+(?:_\d+)*' + Exponent -Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') -Number = group(Imagnumber, Floatnumber, Intnumber) - -# Tail end of ' string. -Single = r"[^'\\]*(?:\\.[^'\\]*)*'" -# Tail end of " string. -Double = r'[^"\\]*(?:\\.[^"\\]*)*"' -# Tail end of ''' string. -Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" -# Tail end of """ string. -Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -_litprefix = r"(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?" -Triple = group(_litprefix + "'''", _litprefix + '"""') -# Single-line ' or " string. -String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') - -# Because of leftmost-then-longest match semantics, be sure to put the -# longest operators first (e.g., if = came before ==, == would get -# recognized as two instances of =). -Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", - r"//=?", r"->", - r"[+\-*/%&@|^=<>]=?", - r"~") - -Bracket = '[][(){}]' -Special = group(r'\r?\n', r':=', r'[:;.,`@]') -Funny = group(Operator, Bracket, Special) - -PlainToken = group(Number, Funny, String, Name) -Token = Ignore + PlainToken - -# First (or only) line of ' or " string. -ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + - group("'", r'\\\r?\n'), - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + - group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) -PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) - -tokenprog, pseudoprog, single3prog, double3prog = map( - re.compile, (Token, PseudoToken, Single3, Double3)) - -_strprefixes = ( - _combinations('r', 'R', 'f', 'F') | - _combinations('r', 'R', 'b', 'B') | - {'u', 'U', 'ur', 'uR', 'Ur', 'UR'} -) - -endprogs = {"'": re.compile(Single), '"': re.compile(Double), - "'''": single3prog, '"""': double3prog, - **{f"{prefix}'''": single3prog for prefix in _strprefixes}, - **{f'{prefix}"""': double3prog for prefix in _strprefixes}, - **{prefix: None for prefix in _strprefixes}} - -triple_quoted = ( - {"'''", '"""'} | - {f"{prefix}'''" for prefix in _strprefixes} | - {f'{prefix}"""' for prefix in _strprefixes} -) -single_quoted = ( - {"'", '"'} | - {f"{prefix}'" for prefix in _strprefixes} | - {f'{prefix}"' for prefix in _strprefixes} -) - -tabsize = 8 - -class TokenError(Exception): pass - -class StopTokenizing(Exception): pass - -def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing - (srow, scol) = xxx_todo_changeme - (erow, ecol) = xxx_todo_changeme1 - print("%d,%d-%d,%d:\t%s\t%s" % \ - (srow, scol, erow, ecol, tok_name[type], repr(token))) - -def tokenize(readline, tokeneater=printtoken): - """ - The tokenize() function accepts two parameters: one representing the - input stream, and one providing an output mechanism for tokenize(). - - The first parameter, readline, must be a callable object which provides - the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. - - The second parameter, tokeneater, must also be a callable object. It is - called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). - """ - try: - tokenize_loop(readline, tokeneater) - except StopTokenizing: - pass - -# backwards compatible interface -def tokenize_loop(readline, tokeneater): - for token_info in generate_tokens(readline): - tokeneater(*token_info) - -class Untokenizer: - - def __init__(self): - self.tokens = [] - self.prev_row = 1 - self.prev_col = 0 - - def add_whitespace(self, start): - row, col = start - assert row <= self.prev_row - col_offset = col - self.prev_col - if col_offset: - self.tokens.append(" " * col_offset) - - def untokenize(self, iterable): - for t in iterable: - if len(t) == 2: - self.compat(t, iterable) - break - tok_type, token, start, end, line = t - self.add_whitespace(start) - self.tokens.append(token) - self.prev_row, self.prev_col = end - if tok_type in (NEWLINE, NL): - self.prev_row += 1 - self.prev_col = 0 - return "".join(self.tokens) - - def compat(self, token, iterable): - startline = False - indents = [] - toks_append = self.tokens.append - toknum, tokval = token - if toknum in (NAME, NUMBER): - tokval += ' ' - if toknum in (NEWLINE, NL): - startline = True - for tok in iterable: - toknum, tokval = tok[:2] - - if toknum in (NAME, NUMBER, ASYNC, AWAIT): - tokval += ' ' - - if toknum == INDENT: - indents.append(tokval) - continue - elif toknum == DEDENT: - indents.pop() - continue - elif toknum in (NEWLINE, NL): - startline = True - elif startline and indents: - toks_append(indents[-1]) - startline = False - toks_append(tokval) - -cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) - -def _get_normal_name(orig_enc): - """Imitates get_normal_name in tokenizer.c.""" - # Only care about the first 12 characters. - enc = orig_enc[:12].lower().replace("_", "-") - if enc == "utf-8" or enc.startswith("utf-8-"): - return "utf-8" - if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ - enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): - return "iso-8859-1" - return orig_enc - -def detect_encoding(readline): - """ - The detect_encoding() function is used to detect the encoding that should - be used to decode a Python source file. It requires one argument, readline, - in the same way as the tokenize() generator. - - It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (left as bytes) it has read - in. - - It detects the encoding from the presence of a utf-8 bom or an encoding - cookie as specified in pep-0263. If both a bom and a cookie are present, but - disagree, a SyntaxError will be raised. If the encoding cookie is an invalid - charset, raise a SyntaxError. Note that if a utf-8 bom is found, - 'utf-8-sig' is returned. - - If no encoding is specified, then the default of 'utf-8' will be returned. - """ - bom_found = False - encoding = None - default = 'utf-8' - def read_or_stop(): - try: - return readline() - except StopIteration: - return bytes() - - def find_cookie(line): - try: - line_string = line.decode('ascii') - except UnicodeDecodeError: - return None - match = cookie_re.match(line_string) - if not match: - return None - encoding = _get_normal_name(match.group(1)) - try: - codec = lookup(encoding) - except LookupError: - # This behaviour mimics the Python interpreter - raise SyntaxError("unknown encoding: " + encoding) - - if bom_found: - if codec.name != 'utf-8': - # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') - encoding += '-sig' - return encoding - - first = read_or_stop() - if first.startswith(BOM_UTF8): - bom_found = True - first = first[3:] - default = 'utf-8-sig' - if not first: - return default, [] - - encoding = find_cookie(first) - if encoding: - return encoding, [first] - if not blank_re.match(first): - return default, [first] - - second = read_or_stop() - if not second: - return default, [first] - - encoding = find_cookie(second) - if encoding: - return encoding, [first, second] - - return default, [first, second] - -def untokenize(iterable): - """Transform tokens back into Python source code. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output text will tokenize the back to the input - t1 = [tok[:2] for tok in generate_tokens(f.readline)] - newcode = untokenize(t1) - readline = iter(newcode.splitlines(1)).next - t2 = [tok[:2] for tokin generate_tokens(readline)] - assert t1 == t2 - """ - ut = Untokenizer() - return ut.untokenize(iterable) - -def generate_tokens(readline): - """ - The generate_tokens() generator requires one argument, readline, which - must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as a string. Alternately, readline - can be a callable function terminating with StopIteration: - readline = open(myfile).next # Example of alternate readline - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple (srow, scol) of ints specifying the row and - column where the token begins in the source; a 2-tuple (erow, ecol) of - ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the - physical line. - """ - lnum = parenlev = continued = 0 - contstr, needcont = '', 0 - contline = None - indents = [0] - - # 'stashed' and 'async_*' are used for async/await parsing - stashed = None - async_def = False - async_def_indent = 0 - async_def_nl = False - - while 1: # loop over lines in stream - try: - line = readline() - except StopIteration: - line = '' - lnum = lnum + 1 - pos, max = 0, len(line) - - if contstr: # continued string - if not line: - raise TokenError("EOF in multi-line string", strstart) - endmatch = endprog.match(line) - if endmatch: - pos = end = endmatch.end(0) - yield (STRING, contstr + line[:end], - strstart, (lnum, end), contline + line) - contstr, needcont = '', 0 - contline = None - elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield (ERRORTOKEN, contstr + line, - strstart, (lnum, len(line)), contline) - contstr = '' - contline = None - continue - else: - contstr = contstr + line - contline = contline + line - continue - - elif parenlev == 0 and not continued: # new statement - if not line: break - column = 0 - while pos < max: # measure leading whitespace - if line[pos] == ' ': column = column + 1 - elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize - elif line[pos] == '\f': column = 0 - else: break - pos = pos + 1 - if pos == max: break - - if stashed: - yield stashed - stashed = None - - if line[pos] in '#\r\n': # skip comments or blank lines - if line[pos] == '#': - comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) - yield (COMMENT, comment_token, - (lnum, pos), (lnum, pos + len(comment_token)), line) - yield (NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], - (lnum, pos), (lnum, len(line)), line) - continue - - if column > indents[-1]: # count indents or dedents - indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - while column < indents[-1]: - if column not in indents: - raise IndentationError( - "unindent does not match any outer indentation level", - ("", lnum, pos, line)) - indents = indents[:-1] - - if async_def and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) - - if async_def and async_def_nl and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - - else: # continued statement - if not line: - raise TokenError("EOF in multi-line statement", (lnum, 0)) - continued = 0 - - while pos < max: - pseudomatch = pseudoprog.match(line, pos) - if pseudomatch: # scan for tokens - start, end = pseudomatch.span(1) - spos, epos, pos = (lnum, start), (lnum, end), end - token, initial = line[start:end], line[start] - - if initial in string.digits or \ - (initial == '.' and token != '.'): # ordinary number - yield (NUMBER, token, spos, epos, line) - elif initial in '\r\n': - newline = NEWLINE - if parenlev > 0: - newline = NL - elif async_def: - async_def_nl = True - if stashed: - yield stashed - stashed = None - yield (newline, token, spos, epos, line) - - elif initial == '#': - assert not token.endswith("\n") - if stashed: - yield stashed - stashed = None - yield (COMMENT, token, spos, epos, line) - elif token in triple_quoted: - endprog = endprogs[token] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - if stashed: - yield stashed - stashed = None - yield (STRING, token, spos, (lnum, pos), line) - else: - strstart = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: - if token[-1] == '\n': # continued string - strstart = (lnum, start) - endprog = (endprogs[initial] or endprogs[token[1]] or - endprogs[token[2]]) - contstr, needcont = line[start:], 1 - contline = line - break - else: # ordinary string - if stashed: - yield stashed - stashed = None - yield (STRING, token, spos, epos, line) - elif initial.isidentifier(): # ordinary name - if token in ('async', 'await'): - if async_def: - yield (ASYNC if token == 'async' else AWAIT, - token, spos, epos, line) - continue - - tok = (NAME, token, spos, epos, line) - if token == 'async' and not stashed: - stashed = tok - continue - - if token in ('def', 'for'): - if (stashed - and stashed[0] == NAME - and stashed[1] == 'async'): - - if token == 'def': - async_def = True - async_def_indent = indents[-1] - - yield (ASYNC, stashed[1], - stashed[2], stashed[3], - stashed[4]) - stashed = None - - if stashed: - yield stashed - stashed = None - - yield tok - elif initial == '\\': # continued stmt - # This yield is new; needed for better idempotency: - if stashed: - yield stashed - stashed = None - yield (NL, token, spos, (lnum, pos), line) - continued = 1 - else: - if initial in '([{': parenlev = parenlev + 1 - elif initial in ')]}': parenlev = parenlev - 1 - if stashed: - yield stashed - stashed = None - yield (OP, token, spos, epos, line) - else: - yield (ERRORTOKEN, line[pos], - (lnum, pos), (lnum, pos+1), line) - pos = pos + 1 - - if stashed: - yield stashed - stashed = None - - for indent in indents[1:]: # pop remaining indent levels - yield (DEDENT, '', (lnum, 0), (lnum, 0), '') - yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') - -if __name__ == '__main__': # testing - import sys - if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) - else: tokenize(sys.stdin.readline) diff --git a/Lib/lib2to3/pygram.py b/Lib/lib2to3/pygram.py deleted file mode 100644 index 24d9db9217f131..00000000000000 --- a/Lib/lib2to3/pygram.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Export the Python grammar and symbols.""" - -# Python imports -import os - -# Local imports -from .pgen2 import token -from .pgen2 import driver -from . import pytree - -# The grammar file -_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt") -_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), - "PatternGrammar.txt") - - -class Symbols(object): - - def __init__(self, grammar): - """Initializer. - - Creates an attribute for each grammar symbol (nonterminal), - whose value is the symbol's type (an int >= 256). - """ - for name, symbol in grammar.symbol2number.items(): - setattr(self, name, symbol) - - -python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE) - -python_symbols = Symbols(python_grammar) - -python_grammar_no_print_statement = python_grammar.copy() -del python_grammar_no_print_statement.keywords["print"] - -python_grammar_no_print_and_exec_statement = python_grammar_no_print_statement.copy() -del python_grammar_no_print_and_exec_statement.keywords["exec"] - -pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE) -pattern_symbols = Symbols(pattern_grammar) diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py deleted file mode 100644 index 729023df0284e1..00000000000000 --- a/Lib/lib2to3/pytree.py +++ /dev/null @@ -1,853 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -""" -Python parse tree definitions. - -This is a very concrete parse tree; we need to keep every token and -even the comments and whitespace between tokens. - -There's also a pattern matching implementation here. -""" - -__author__ = "Guido van Rossum " - -import sys -from io import StringIO - -HUGE = 0x7FFFFFFF # maximum repeat count, default max - -_type_reprs = {} -def type_repr(type_num): - global _type_reprs - if not _type_reprs: - from .pygram import python_symbols - # printing tokens is possible but not as useful - # from .pgen2 import token // token.__dict__.items(): - for name, val in python_symbols.__dict__.items(): - if type(val) == int: _type_reprs[val] = name - return _type_reprs.setdefault(type_num, type_num) - -class Base(object): - - """ - Abstract base class for Node and Leaf. - - This provides some default functionality and boilerplate using the - template pattern. - - A node may be a subnode of at most one parent. - """ - - # Default values for instance variables - type = None # int: token number (< 256) or symbol number (>= 256) - parent = None # Parent node pointer, or None - children = () # Tuple of subnodes - was_changed = False - was_checked = False - - def __new__(cls, *args, **kwds): - """Constructor that prevents Base from being instantiated.""" - assert cls is not Base, "Cannot instantiate Base" - return object.__new__(cls) - - def __eq__(self, other): - """ - Compare two nodes for equality. - - This calls the method _eq(). - """ - if self.__class__ is not other.__class__: - return NotImplemented - return self._eq(other) - - __hash__ = None # For Py3 compatibility. - - def _eq(self, other): - """ - Compare two nodes for equality. - - This is called by __eq__ and __ne__. It is only called if the two nodes - have the same type. This must be implemented by the concrete subclass. - Nodes should be considered equal if they have the same structure, - ignoring the prefix string and other context information. - """ - raise NotImplementedError - - def clone(self): - """ - Return a cloned (deep) copy of self. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def post_order(self): - """ - Return a post-order iterator for the tree. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def pre_order(self): - """ - Return a pre-order iterator for the tree. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def replace(self, new): - """Replace this node with a new one in the parent.""" - assert self.parent is not None, str(self) - assert new is not None - if not isinstance(new, list): - new = [new] - l_children = [] - found = False - for ch in self.parent.children: - if ch is self: - assert not found, (self.parent.children, self, new) - if new is not None: - l_children.extend(new) - found = True - else: - l_children.append(ch) - assert found, (self.children, self, new) - self.parent.changed() - self.parent.children = l_children - for x in new: - x.parent = self.parent - self.parent = None - - def get_lineno(self): - """Return the line number which generated the invocant node.""" - node = self - while not isinstance(node, Leaf): - if not node.children: - return - node = node.children[0] - return node.lineno - - def changed(self): - if self.parent: - self.parent.changed() - self.was_changed = True - - def remove(self): - """ - Remove the node from the tree. Returns the position of the node in its - parent's children before it was removed. - """ - if self.parent: - for i, node in enumerate(self.parent.children): - if node is self: - self.parent.changed() - del self.parent.children[i] - self.parent = None - return i - - @property - def next_sibling(self): - """ - The node immediately following the invocant in their parent's children - list. If the invocant does not have a next sibling, it is None - """ - if self.parent is None: - return None - - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - try: - return self.parent.children[i+1] - except IndexError: - return None - - @property - def prev_sibling(self): - """ - The node immediately preceding the invocant in their parent's children - list. If the invocant does not have a previous sibling, it is None. - """ - if self.parent is None: - return None - - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - if i == 0: - return None - return self.parent.children[i-1] - - def leaves(self): - for child in self.children: - yield from child.leaves() - - def depth(self): - if self.parent is None: - return 0 - return 1 + self.parent.depth() - - def get_suffix(self): - """ - Return the string immediately following the invocant node. This is - effectively equivalent to node.next_sibling.prefix - """ - next_sib = self.next_sibling - if next_sib is None: - return "" - return next_sib.prefix - - if sys.version_info < (3, 0): - def __str__(self): - return str(self).encode("ascii") - -class Node(Base): - - """Concrete implementation for interior nodes.""" - - def __init__(self,type, children, - context=None, - prefix=None, - fixers_applied=None): - """ - Initializer. - - Takes a type constant (a symbol number >= 256), a sequence of - child nodes, and an optional context keyword argument. - - As a side effect, the parent pointers of the children are updated. - """ - assert type >= 256, type - self.type = type - self.children = list(children) - for ch in self.children: - assert ch.parent is None, repr(ch) - ch.parent = self - if prefix is not None: - self.prefix = prefix - if fixers_applied: - self.fixers_applied = fixers_applied[:] - else: - self.fixers_applied = None - - def __repr__(self): - """Return a canonical string representation.""" - return "%s(%s, %r)" % (self.__class__.__name__, - type_repr(self.type), - self.children) - - def __unicode__(self): - """ - Return a pretty string representation. - - This reproduces the input source exactly. - """ - return "".join(map(str, self.children)) - - if sys.version_info > (3, 0): - __str__ = __unicode__ - - def _eq(self, other): - """Compare two nodes for equality.""" - return (self.type, self.children) == (other.type, other.children) - - def clone(self): - """Return a cloned (deep) copy of self.""" - return Node(self.type, [ch.clone() for ch in self.children], - fixers_applied=self.fixers_applied) - - def post_order(self): - """Return a post-order iterator for the tree.""" - for child in self.children: - yield from child.post_order() - yield self - - def pre_order(self): - """Return a pre-order iterator for the tree.""" - yield self - for child in self.children: - yield from child.pre_order() - - @property - def prefix(self): - """ - The whitespace and comments preceding this node in the input. - """ - if not self.children: - return "" - return self.children[0].prefix - - @prefix.setter - def prefix(self, prefix): - if self.children: - self.children[0].prefix = prefix - - def set_child(self, i, child): - """ - Equivalent to 'node.children[i] = child'. This method also sets the - child's parent attribute appropriately. - """ - child.parent = self - self.children[i].parent = None - self.children[i] = child - self.changed() - - def insert_child(self, i, child): - """ - Equivalent to 'node.children.insert(i, child)'. This method also sets - the child's parent attribute appropriately. - """ - child.parent = self - self.children.insert(i, child) - self.changed() - - def append_child(self, child): - """ - Equivalent to 'node.children.append(child)'. This method also sets the - child's parent attribute appropriately. - """ - child.parent = self - self.children.append(child) - self.changed() - - -class Leaf(Base): - - """Concrete implementation for leaf nodes.""" - - # Default values for instance variables - _prefix = "" # Whitespace and comments preceding this token in the input - lineno = 0 # Line where this token starts in the input - column = 0 # Column where this token tarts in the input - - def __init__(self, type, value, - context=None, - prefix=None, - fixers_applied=[]): - """ - Initializer. - - Takes a type constant (a token number < 256), a string value, and an - optional context keyword argument. - """ - assert 0 <= type < 256, type - if context is not None: - self._prefix, (self.lineno, self.column) = context - self.type = type - self.value = value - if prefix is not None: - self._prefix = prefix - self.fixers_applied = fixers_applied[:] - - def __repr__(self): - """Return a canonical string representation.""" - return "%s(%r, %r)" % (self.__class__.__name__, - self.type, - self.value) - - def __unicode__(self): - """ - Return a pretty string representation. - - This reproduces the input source exactly. - """ - return self.prefix + str(self.value) - - if sys.version_info > (3, 0): - __str__ = __unicode__ - - def _eq(self, other): - """Compare two nodes for equality.""" - return (self.type, self.value) == (other.type, other.value) - - def clone(self): - """Return a cloned (deep) copy of self.""" - return Leaf(self.type, self.value, - (self.prefix, (self.lineno, self.column)), - fixers_applied=self.fixers_applied) - - def leaves(self): - yield self - - def post_order(self): - """Return a post-order iterator for the tree.""" - yield self - - def pre_order(self): - """Return a pre-order iterator for the tree.""" - yield self - - @property - def prefix(self): - """ - The whitespace and comments preceding this token in the input. - """ - return self._prefix - - @prefix.setter - def prefix(self, prefix): - self.changed() - self._prefix = prefix - -def convert(gr, raw_node): - """ - Convert raw node information to a Node or Leaf instance. - - This is passed to the parser driver which calls it whenever a reduction of a - grammar rule produces a new complete node, so that the tree is build - strictly bottom-up. - """ - type, value, context, children = raw_node - if children or type in gr.number2symbol: - # If there's exactly one child, return that child instead of - # creating a new node. - if len(children) == 1: - return children[0] - return Node(type, children, context=context) - else: - return Leaf(type, value, context=context) - - -class BasePattern(object): - - """ - A pattern is a tree matching pattern. - - It looks for a specific node type (token or symbol), and - optionally for a specific content. - - This is an abstract base class. There are three concrete - subclasses: - - - LeafPattern matches a single leaf node; - - NodePattern matches a single node (usually non-leaf); - - WildcardPattern matches a sequence of nodes of variable length. - """ - - # Defaults for instance variables - type = None # Node type (token if < 256, symbol if >= 256) - content = None # Optional content matching pattern - name = None # Optional name used to store match in results dict - - def __new__(cls, *args, **kwds): - """Constructor that prevents BasePattern from being instantiated.""" - assert cls is not BasePattern, "Cannot instantiate BasePattern" - return object.__new__(cls) - - def __repr__(self): - args = [type_repr(self.type), self.content, self.name] - while args and args[-1] is None: - del args[-1] - return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args))) - - def optimize(self): - """ - A subclass can define this as a hook for optimizations. - - Returns either self or another node with the same effect. - """ - return self - - def match(self, node, results=None): - """ - Does this pattern exactly match a node? - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - Default implementation for non-wildcard patterns. - """ - if self.type is not None and node.type != self.type: - return False - if self.content is not None: - r = None - if results is not None: - r = {} - if not self._submatch(node, r): - return False - if r: - results.update(r) - if results is not None and self.name: - results[self.name] = node - return True - - def match_seq(self, nodes, results=None): - """ - Does this pattern exactly match a sequence of nodes? - - Default implementation for non-wildcard patterns. - """ - if len(nodes) != 1: - return False - return self.match(nodes[0], results) - - def generate_matches(self, nodes): - """ - Generator yielding all matches for this pattern. - - Default implementation for non-wildcard patterns. - """ - r = {} - if nodes and self.match(nodes[0], r): - yield 1, r - - -class LeafPattern(BasePattern): - - def __init__(self, type=None, content=None, name=None): - """ - Initializer. Takes optional type, content, and name. - - The type, if given must be a token type (< 256). If not given, - this matches any *leaf* node; the content may still be required. - - The content, if given, must be a string. - - If a name is given, the matching node is stored in the results - dict under that key. - """ - if type is not None: - assert 0 <= type < 256, type - if content is not None: - assert isinstance(content, str), repr(content) - self.type = type - self.content = content - self.name = name - - def match(self, node, results=None): - """Override match() to insist on a leaf node.""" - if not isinstance(node, Leaf): - return False - return BasePattern.match(self, node, results) - - def _submatch(self, node, results=None): - """ - Match the pattern's content to the node's children. - - This assumes the node type matches and self.content is not None. - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - When returning False, the results dict may still be updated. - """ - return self.content == node.value - - -class NodePattern(BasePattern): - - wildcards = False - - def __init__(self, type=None, content=None, name=None): - """ - Initializer. Takes optional type, content, and name. - - The type, if given, must be a symbol type (>= 256). If the - type is None this matches *any* single node (leaf or not), - except if content is not None, in which it only matches - non-leaf nodes that also match the content pattern. - - The content, if not None, must be a sequence of Patterns that - must match the node's children exactly. If the content is - given, the type must not be None. - - If a name is given, the matching node is stored in the results - dict under that key. - """ - if type is not None: - assert type >= 256, type - if content is not None: - assert not isinstance(content, str), repr(content) - content = list(content) - for i, item in enumerate(content): - assert isinstance(item, BasePattern), (i, item) - if isinstance(item, WildcardPattern): - self.wildcards = True - self.type = type - self.content = content - self.name = name - - def _submatch(self, node, results=None): - """ - Match the pattern's content to the node's children. - - This assumes the node type matches and self.content is not None. - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - When returning False, the results dict may still be updated. - """ - if self.wildcards: - for c, r in generate_matches(self.content, node.children): - if c == len(node.children): - if results is not None: - results.update(r) - return True - return False - if len(self.content) != len(node.children): - return False - for subpattern, child in zip(self.content, node.children): - if not subpattern.match(child, results): - return False - return True - - -class WildcardPattern(BasePattern): - - """ - A wildcard pattern can match zero or more nodes. - - This has all the flexibility needed to implement patterns like: - - .* .+ .? .{m,n} - (a b c | d e | f) - (...)* (...)+ (...)? (...){m,n} - - except it always uses non-greedy matching. - """ - - def __init__(self, content=None, min=0, max=HUGE, name=None): - """ - Initializer. - - Args: - content: optional sequence of subsequences of patterns; - if absent, matches one node; - if present, each subsequence is an alternative [*] - min: optional minimum number of times to match, default 0 - max: optional maximum number of times to match, default HUGE - name: optional name assigned to this match - - [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is - equivalent to (a b c | d e | f g h); if content is None, - this is equivalent to '.' in regular expression terms. - The min and max parameters work as follows: - min=0, max=maxint: .* - min=1, max=maxint: .+ - min=0, max=1: .? - min=1, max=1: . - If content is not None, replace the dot with the parenthesized - list of alternatives, e.g. (a b c | d e | f g h)* - """ - assert 0 <= min <= max <= HUGE, (min, max) - if content is not None: - content = tuple(map(tuple, content)) # Protect against alterations - # Check sanity of alternatives - assert len(content), repr(content) # Can't have zero alternatives - for alt in content: - assert len(alt), repr(alt) # Can have empty alternatives - self.content = content - self.min = min - self.max = max - self.name = name - - def optimize(self): - """Optimize certain stacked wildcard patterns.""" - subpattern = None - if (self.content is not None and - len(self.content) == 1 and len(self.content[0]) == 1): - subpattern = self.content[0][0] - if self.min == 1 and self.max == 1: - if self.content is None: - return NodePattern(name=self.name) - if subpattern is not None and self.name == subpattern.name: - return subpattern.optimize() - if (self.min <= 1 and isinstance(subpattern, WildcardPattern) and - subpattern.min <= 1 and self.name == subpattern.name): - return WildcardPattern(subpattern.content, - self.min*subpattern.min, - self.max*subpattern.max, - subpattern.name) - return self - - def match(self, node, results=None): - """Does this pattern exactly match a node?""" - return self.match_seq([node], results) - - def match_seq(self, nodes, results=None): - """Does this pattern exactly match a sequence of nodes?""" - for c, r in self.generate_matches(nodes): - if c == len(nodes): - if results is not None: - results.update(r) - if self.name: - results[self.name] = list(nodes) - return True - return False - - def generate_matches(self, nodes): - """ - Generator yielding matches for a sequence of nodes. - - Args: - nodes: sequence of nodes - - Yields: - (count, results) tuples where: - count: the match comprises nodes[:count]; - results: dict containing named submatches. - """ - if self.content is None: - # Shortcut for special case (see __init__.__doc__) - for count in range(self.min, 1 + min(len(nodes), self.max)): - r = {} - if self.name: - r[self.name] = nodes[:count] - yield count, r - elif self.name == "bare_name": - yield self._bare_name_matches(nodes) - else: - # The reason for this is that hitting the recursion limit usually - # results in some ugly messages about how RuntimeErrors are being - # ignored. We only have to do this on CPython, though, because other - # implementations don't have this nasty bug in the first place. - if hasattr(sys, "getrefcount"): - save_stderr = sys.stderr - sys.stderr = StringIO() - try: - for count, r in self._recursive_matches(nodes, 0): - if self.name: - r[self.name] = nodes[:count] - yield count, r - except RuntimeError: - # Fall back to the iterative pattern matching scheme if the - # recursive scheme hits the recursion limit (RecursionError). - for count, r in self._iterative_matches(nodes): - if self.name: - r[self.name] = nodes[:count] - yield count, r - finally: - if hasattr(sys, "getrefcount"): - sys.stderr = save_stderr - - def _iterative_matches(self, nodes): - """Helper to iteratively yield the matches.""" - nodelen = len(nodes) - if 0 >= self.min: - yield 0, {} - - results = [] - # generate matches that use just one alt from self.content - for alt in self.content: - for c, r in generate_matches(alt, nodes): - yield c, r - results.append((c, r)) - - # for each match, iterate down the nodes - while results: - new_results = [] - for c0, r0 in results: - # stop if the entire set of nodes has been matched - if c0 < nodelen and c0 <= self.max: - for alt in self.content: - for c1, r1 in generate_matches(alt, nodes[c0:]): - if c1 > 0: - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r - new_results.append((c0 + c1, r)) - results = new_results - - def _bare_name_matches(self, nodes): - """Special optimized matcher for bare_name.""" - count = 0 - r = {} - done = False - max = len(nodes) - while not done and count < max: - done = True - for leaf in self.content: - if leaf[0].match(nodes[count], r): - count += 1 - done = False - break - r[self.name] = nodes[:count] - return count, r - - def _recursive_matches(self, nodes, count): - """Helper to recursively yield the matches.""" - assert self.content is not None - if count >= self.min: - yield 0, {} - if count < self.max: - for alt in self.content: - for c0, r0 in generate_matches(alt, nodes): - for c1, r1 in self._recursive_matches(nodes[c0:], count+1): - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r - - -class NegatedPattern(BasePattern): - - def __init__(self, content=None): - """ - Initializer. - - The argument is either a pattern or None. If it is None, this - only matches an empty sequence (effectively '$' in regex - lingo). If it is not None, this matches whenever the argument - pattern doesn't have any matches. - """ - if content is not None: - assert isinstance(content, BasePattern), repr(content) - self.content = content - - def match(self, node): - # We never match a node in its entirety - return False - - def match_seq(self, nodes): - # We only match an empty sequence of nodes in its entirety - return len(nodes) == 0 - - def generate_matches(self, nodes): - if self.content is None: - # Return a match if there is an empty sequence - if len(nodes) == 0: - yield 0, {} - else: - # Return a match if the argument pattern has no matches - for c, r in self.content.generate_matches(nodes): - return - yield 0, {} - - -def generate_matches(patterns, nodes): - """ - Generator yielding matches for a sequence of patterns and nodes. - - Args: - patterns: a sequence of patterns - nodes: a sequence of nodes - - Yields: - (count, results) tuples where: - count: the entire sequence of patterns matches nodes[:count]; - results: dict containing named submatches. - """ - if not patterns: - yield 0, {} - else: - p, rest = patterns[0], patterns[1:] - for c0, r0 in p.generate_matches(nodes): - if not rest: - yield c0, r0 - else: - for c1, r1 in generate_matches(rest, nodes[c0:]): - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py deleted file mode 100644 index 3a5aafffc6df07..00000000000000 --- a/Lib/lib2to3/refactor.py +++ /dev/null @@ -1,732 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Refactoring framework. - -Used as a main program, this can refactor any number of files and/or -recursively descend down directories. Imported as a module, this -provides infrastructure to write your own refactoring tool. -""" - -__author__ = "Guido van Rossum " - - -# Python imports -import io -import os -import pkgutil -import sys -import logging -import operator -import collections -from itertools import chain - -# Local imports -from .pgen2 import driver, tokenize, token -from .fixer_util import find_root -from . import pytree, pygram -from . import btm_matcher as bm - - -def get_all_fix_names(fixer_pkg, remove_prefix=True): - """Return a sorted list of all available fix names in the given package.""" - pkg = __import__(fixer_pkg, [], [], ["*"]) - fix_names = [] - for finder, name, ispkg in pkgutil.iter_modules(pkg.__path__): - if name.startswith("fix_"): - if remove_prefix: - name = name[4:] - fix_names.append(name) - return fix_names - - -class _EveryNode(Exception): - pass - - -def _get_head_types(pat): - """ Accepts a pytree Pattern Node and returns a set - of the pattern types which will match first. """ - - if isinstance(pat, (pytree.NodePattern, pytree.LeafPattern)): - # NodePatters must either have no type and no content - # or a type and content -- so they don't get any farther - # Always return leafs - if pat.type is None: - raise _EveryNode - return {pat.type} - - if isinstance(pat, pytree.NegatedPattern): - if pat.content: - return _get_head_types(pat.content) - raise _EveryNode # Negated Patterns don't have a type - - if isinstance(pat, pytree.WildcardPattern): - # Recurse on each node in content - r = set() - for p in pat.content: - for x in p: - r.update(_get_head_types(x)) - return r - - raise Exception("Oh no! I don't understand pattern %s" %(pat)) - - -def _get_headnode_dict(fixer_list): - """ Accepts a list of fixers and returns a dictionary - of head node type --> fixer list. """ - head_nodes = collections.defaultdict(list) - every = [] - for fixer in fixer_list: - if fixer.pattern: - try: - heads = _get_head_types(fixer.pattern) - except _EveryNode: - every.append(fixer) - else: - for node_type in heads: - head_nodes[node_type].append(fixer) - else: - if fixer._accept_type is not None: - head_nodes[fixer._accept_type].append(fixer) - else: - every.append(fixer) - for node_type in chain(pygram.python_grammar.symbol2number.values(), - pygram.python_grammar.tokens): - head_nodes[node_type].extend(every) - return dict(head_nodes) - - -def get_fixers_from_package(pkg_name): - """ - Return the fully qualified names for fixers in the package pkg_name. - """ - return [pkg_name + "." + fix_name - for fix_name in get_all_fix_names(pkg_name, False)] - -def _identity(obj): - return obj - - -def _detect_future_features(source): - have_docstring = False - gen = tokenize.generate_tokens(io.StringIO(source).readline) - def advance(): - tok = next(gen) - return tok[0], tok[1] - ignore = frozenset({token.NEWLINE, tokenize.NL, token.COMMENT}) - features = set() - try: - while True: - tp, value = advance() - if tp in ignore: - continue - elif tp == token.STRING: - if have_docstring: - break - have_docstring = True - elif tp == token.NAME and value == "from": - tp, value = advance() - if tp != token.NAME or value != "__future__": - break - tp, value = advance() - if tp != token.NAME or value != "import": - break - tp, value = advance() - if tp == token.OP and value == "(": - tp, value = advance() - while tp == token.NAME: - features.add(value) - tp, value = advance() - if tp != token.OP or value != ",": - break - tp, value = advance() - else: - break - except StopIteration: - pass - return frozenset(features) - - -class FixerError(Exception): - """A fixer could not be loaded.""" - - -class RefactoringTool(object): - - _default_options = {"print_function" : False, - "exec_function": False, - "write_unchanged_files" : False} - - CLASS_PREFIX = "Fix" # The prefix for fixer classes - FILE_PREFIX = "fix_" # The prefix for modules with a fixer within - - def __init__(self, fixer_names, options=None, explicit=None): - """Initializer. - - Args: - fixer_names: a list of fixers to import - options: a dict with configuration. - explicit: a list of fixers to run even if they are explicit. - """ - self.fixers = fixer_names - self.explicit = explicit or [] - self.options = self._default_options.copy() - if options is not None: - self.options.update(options) - self.grammar = pygram.python_grammar.copy() - - if self.options['print_function']: - del self.grammar.keywords["print"] - elif self.options['exec_function']: - del self.grammar.keywords["exec"] - - # When this is True, the refactor*() methods will call write_file() for - # files processed even if they were not changed during refactoring. If - # and only if the refactor method's write parameter was True. - self.write_unchanged_files = self.options.get("write_unchanged_files") - self.errors = [] - self.logger = logging.getLogger("RefactoringTool") - self.fixer_log = [] - self.wrote = False - self.driver = driver.Driver(self.grammar, - convert=pytree.convert, - logger=self.logger) - self.pre_order, self.post_order = self.get_fixers() - - - self.files = [] # List of files that were or should be modified - - self.BM = bm.BottomMatcher() - self.bmi_pre_order = [] # Bottom Matcher incompatible fixers - self.bmi_post_order = [] - - for fixer in chain(self.post_order, self.pre_order): - if fixer.BM_compatible: - self.BM.add_fixer(fixer) - # remove fixers that will be handled by the bottom-up - # matcher - elif fixer in self.pre_order: - self.bmi_pre_order.append(fixer) - elif fixer in self.post_order: - self.bmi_post_order.append(fixer) - - self.bmi_pre_order_heads = _get_headnode_dict(self.bmi_pre_order) - self.bmi_post_order_heads = _get_headnode_dict(self.bmi_post_order) - - - - def get_fixers(self): - """Inspects the options to load the requested patterns and handlers. - - Returns: - (pre_order, post_order), where pre_order is the list of fixers that - want a pre-order AST traversal, and post_order is the list that want - post-order traversal. - """ - pre_order_fixers = [] - post_order_fixers = [] - for fix_mod_path in self.fixers: - mod = __import__(fix_mod_path, {}, {}, ["*"]) - fix_name = fix_mod_path.rsplit(".", 1)[-1] - if fix_name.startswith(self.FILE_PREFIX): - fix_name = fix_name[len(self.FILE_PREFIX):] - parts = fix_name.split("_") - class_name = self.CLASS_PREFIX + "".join([p.title() for p in parts]) - try: - fix_class = getattr(mod, class_name) - except AttributeError: - raise FixerError("Can't find %s.%s" % (fix_name, class_name)) from None - fixer = fix_class(self.options, self.fixer_log) - if fixer.explicit and self.explicit is not True and \ - fix_mod_path not in self.explicit: - self.log_message("Skipping optional fixer: %s", fix_name) - continue - - self.log_debug("Adding transformation: %s", fix_name) - if fixer.order == "pre": - pre_order_fixers.append(fixer) - elif fixer.order == "post": - post_order_fixers.append(fixer) - else: - raise FixerError("Illegal fixer order: %r" % fixer.order) - - key_func = operator.attrgetter("run_order") - pre_order_fixers.sort(key=key_func) - post_order_fixers.sort(key=key_func) - return (pre_order_fixers, post_order_fixers) - - def log_error(self, msg, *args, **kwds): - """Called when an error occurs.""" - raise - - def log_message(self, msg, *args): - """Hook to log a message.""" - if args: - msg = msg % args - self.logger.info(msg) - - def log_debug(self, msg, *args): - if args: - msg = msg % args - self.logger.debug(msg) - - def print_output(self, old_text, new_text, filename, equal): - """Called with the old version, new version, and filename of a - refactored file.""" - pass - - def refactor(self, items, write=False, doctests_only=False): - """Refactor a list of files and directories.""" - - for dir_or_file in items: - if os.path.isdir(dir_or_file): - self.refactor_dir(dir_or_file, write, doctests_only) - else: - self.refactor_file(dir_or_file, write, doctests_only) - - def refactor_dir(self, dir_name, write=False, doctests_only=False): - """Descends down a directory and refactor every Python file found. - - Python files are assumed to have a .py extension. - - Files and subdirectories starting with '.' are skipped. - """ - py_ext = os.extsep + "py" - for dirpath, dirnames, filenames in os.walk(dir_name): - self.log_debug("Descending into %s", dirpath) - dirnames.sort() - filenames.sort() - for name in filenames: - if (not name.startswith(".") and - os.path.splitext(name)[1] == py_ext): - fullname = os.path.join(dirpath, name) - self.refactor_file(fullname, write, doctests_only) - # Modify dirnames in-place to remove subdirs with leading dots - dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")] - - def _read_python_source(self, filename): - """ - Do our best to decode a Python source file correctly. - """ - try: - f = open(filename, "rb") - except OSError as err: - self.log_error("Can't open %s: %s", filename, err) - return None, None - try: - encoding = tokenize.detect_encoding(f.readline)[0] - finally: - f.close() - with io.open(filename, "r", encoding=encoding, newline='') as f: - return f.read(), encoding - - def refactor_file(self, filename, write=False, doctests_only=False): - """Refactors a file.""" - input, encoding = self._read_python_source(filename) - if input is None: - # Reading the file failed. - return - input += "\n" # Silence certain parse errors - if doctests_only: - self.log_debug("Refactoring doctests in %s", filename) - output = self.refactor_docstring(input, filename) - if self.write_unchanged_files or output != input: - self.processed_file(output, filename, input, write, encoding) - else: - self.log_debug("No doctest changes in %s", filename) - else: - tree = self.refactor_string(input, filename) - if self.write_unchanged_files or (tree and tree.was_changed): - # The [:-1] is to take off the \n we added earlier - self.processed_file(str(tree)[:-1], filename, - write=write, encoding=encoding) - else: - self.log_debug("No changes in %s", filename) - - def refactor_string(self, data, name): - """Refactor a given input string. - - Args: - data: a string holding the code to be refactored. - name: a human-readable name for use in error/log messages. - - Returns: - An AST corresponding to the refactored input stream; None if - there were errors during the parse. - """ - features = _detect_future_features(data) - if "print_function" in features: - self.driver.grammar = pygram.python_grammar_no_print_statement - try: - tree = self.driver.parse_string(data) - except Exception as err: - self.log_error("Can't parse %s: %s: %s", - name, err.__class__.__name__, err) - return - finally: - self.driver.grammar = self.grammar - tree.future_features = features - self.log_debug("Refactoring %s", name) - self.refactor_tree(tree, name) - return tree - - def refactor_stdin(self, doctests_only=False): - input = sys.stdin.read() - if doctests_only: - self.log_debug("Refactoring doctests in stdin") - output = self.refactor_docstring(input, "") - if self.write_unchanged_files or output != input: - self.processed_file(output, "", input) - else: - self.log_debug("No doctest changes in stdin") - else: - tree = self.refactor_string(input, "") - if self.write_unchanged_files or (tree and tree.was_changed): - self.processed_file(str(tree), "", input) - else: - self.log_debug("No changes in stdin") - - def refactor_tree(self, tree, name): - """Refactors a parse tree (modifying the tree in place). - - For compatible patterns the bottom matcher module is - used. Otherwise the tree is traversed node-to-node for - matches. - - Args: - tree: a pytree.Node instance representing the root of the tree - to be refactored. - name: a human-readable name for this tree. - - Returns: - True if the tree was modified, False otherwise. - """ - - for fixer in chain(self.pre_order, self.post_order): - fixer.start_tree(tree, name) - - #use traditional matching for the incompatible fixers - self.traverse_by(self.bmi_pre_order_heads, tree.pre_order()) - self.traverse_by(self.bmi_post_order_heads, tree.post_order()) - - # obtain a set of candidate nodes - match_set = self.BM.run(tree.leaves()) - - while any(match_set.values()): - for fixer in self.BM.fixers: - if fixer in match_set and match_set[fixer]: - #sort by depth; apply fixers from bottom(of the AST) to top - match_set[fixer].sort(key=pytree.Base.depth, reverse=True) - - if fixer.keep_line_order: - #some fixers(eg fix_imports) must be applied - #with the original file's line order - match_set[fixer].sort(key=pytree.Base.get_lineno) - - for node in list(match_set[fixer]): - if node in match_set[fixer]: - match_set[fixer].remove(node) - - try: - find_root(node) - except ValueError: - # this node has been cut off from a - # previous transformation ; skip - continue - - if node.fixers_applied and fixer in node.fixers_applied: - # do not apply the same fixer again - continue - - results = fixer.match(node) - - if results: - new = fixer.transform(node, results) - if new is not None: - node.replace(new) - #new.fixers_applied.append(fixer) - for node in new.post_order(): - # do not apply the fixer again to - # this or any subnode - if not node.fixers_applied: - node.fixers_applied = [] - node.fixers_applied.append(fixer) - - # update the original match set for - # the added code - new_matches = self.BM.run(new.leaves()) - for fxr in new_matches: - if not fxr in match_set: - match_set[fxr]=[] - - match_set[fxr].extend(new_matches[fxr]) - - for fixer in chain(self.pre_order, self.post_order): - fixer.finish_tree(tree, name) - return tree.was_changed - - def traverse_by(self, fixers, traversal): - """Traverse an AST, applying a set of fixers to each node. - - This is a helper method for refactor_tree(). - - Args: - fixers: a list of fixer instances. - traversal: a generator that yields AST nodes. - - Returns: - None - """ - if not fixers: - return - for node in traversal: - for fixer in fixers[node.type]: - results = fixer.match(node) - if results: - new = fixer.transform(node, results) - if new is not None: - node.replace(new) - node = new - - def processed_file(self, new_text, filename, old_text=None, write=False, - encoding=None): - """ - Called when a file has been refactored and there may be changes. - """ - self.files.append(filename) - if old_text is None: - old_text = self._read_python_source(filename)[0] - if old_text is None: - return - equal = old_text == new_text - self.print_output(old_text, new_text, filename, equal) - if equal: - self.log_debug("No changes to %s", filename) - if not self.write_unchanged_files: - return - if write: - self.write_file(new_text, filename, old_text, encoding) - else: - self.log_debug("Not writing changes to %s", filename) - - def write_file(self, new_text, filename, old_text, encoding=None): - """Writes a string to a file. - - It first shows a unified diff between the old text and the new text, and - then rewrites the file; the latter is only done if the write option is - set. - """ - try: - fp = io.open(filename, "w", encoding=encoding, newline='') - except OSError as err: - self.log_error("Can't create %s: %s", filename, err) - return - - with fp: - try: - fp.write(new_text) - except OSError as err: - self.log_error("Can't write %s: %s", filename, err) - self.log_debug("Wrote changes to %s", filename) - self.wrote = True - - PS1 = ">>> " - PS2 = "... " - - def refactor_docstring(self, input, filename): - """Refactors a docstring, looking for doctests. - - This returns a modified version of the input string. It looks - for doctests, which start with a ">>>" prompt, and may be - continued with "..." prompts, as long as the "..." is indented - the same as the ">>>". - - (Unfortunately we can't use the doctest module's parser, - since, like most parsers, it is not geared towards preserving - the original source.) - """ - result = [] - block = None - block_lineno = None - indent = None - lineno = 0 - for line in input.splitlines(keepends=True): - lineno += 1 - if line.lstrip().startswith(self.PS1): - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - block_lineno = lineno - block = [line] - i = line.find(self.PS1) - indent = line[:i] - elif (indent is not None and - (line.startswith(indent + self.PS2) or - line == indent + self.PS2.rstrip() + "\n")): - block.append(line) - else: - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - block = None - indent = None - result.append(line) - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - return "".join(result) - - def refactor_doctest(self, block, lineno, indent, filename): - """Refactors one doctest. - - A doctest is given as a block of lines, the first of which starts - with ">>>" (possibly indented), while the remaining lines start - with "..." (identically indented). - - """ - try: - tree = self.parse_block(block, lineno, indent) - except Exception as err: - if self.logger.isEnabledFor(logging.DEBUG): - for line in block: - self.log_debug("Source: %s", line.rstrip("\n")) - self.log_error("Can't parse docstring in %s line %s: %s: %s", - filename, lineno, err.__class__.__name__, err) - return block - if self.refactor_tree(tree, filename): - new = str(tree).splitlines(keepends=True) - # Undo the adjustment of the line numbers in wrap_toks() below. - clipped, new = new[:lineno-1], new[lineno-1:] - assert clipped == ["\n"] * (lineno-1), clipped - if not new[-1].endswith("\n"): - new[-1] += "\n" - block = [indent + self.PS1 + new.pop(0)] - if new: - block += [indent + self.PS2 + line for line in new] - return block - - def summarize(self): - if self.wrote: - were = "were" - else: - were = "need to be" - if not self.files: - self.log_message("No files %s modified.", were) - else: - self.log_message("Files that %s modified:", were) - for file in self.files: - self.log_message(file) - if self.fixer_log: - self.log_message("Warnings/messages while refactoring:") - for message in self.fixer_log: - self.log_message(message) - if self.errors: - if len(self.errors) == 1: - self.log_message("There was 1 error:") - else: - self.log_message("There were %d errors:", len(self.errors)) - for msg, args, kwds in self.errors: - self.log_message(msg, *args, **kwds) - - def parse_block(self, block, lineno, indent): - """Parses a block into a tree. - - This is necessary to get correct line number / offset information - in the parser diagnostics and embedded into the parse tree. - """ - tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent)) - tree.future_features = frozenset() - return tree - - def wrap_toks(self, block, lineno, indent): - """Wraps a tokenize stream to systematically modify start/end.""" - tokens = tokenize.generate_tokens(self.gen_lines(block, indent).__next__) - for type, value, (line0, col0), (line1, col1), line_text in tokens: - line0 += lineno - 1 - line1 += lineno - 1 - # Don't bother updating the columns; this is too complicated - # since line_text would also have to be updated and it would - # still break for tokens spanning lines. Let the user guess - # that the column numbers for doctests are relative to the - # end of the prompt string (PS1 or PS2). - yield type, value, (line0, col0), (line1, col1), line_text - - - def gen_lines(self, block, indent): - """Generates lines as expected by tokenize from a list of lines. - - This strips the first len(indent + self.PS1) characters off each line. - """ - prefix1 = indent + self.PS1 - prefix2 = indent + self.PS2 - prefix = prefix1 - for line in block: - if line.startswith(prefix): - yield line[len(prefix):] - elif line == prefix.rstrip() + "\n": - yield "\n" - else: - raise AssertionError("line=%r, prefix=%r" % (line, prefix)) - prefix = prefix2 - while True: - yield "" - - -class MultiprocessingUnsupported(Exception): - pass - - -class MultiprocessRefactoringTool(RefactoringTool): - - def __init__(self, *args, **kwargs): - super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs) - self.queue = None - self.output_lock = None - - def refactor(self, items, write=False, doctests_only=False, - num_processes=1): - if num_processes == 1: - return super(MultiprocessRefactoringTool, self).refactor( - items, write, doctests_only) - try: - import multiprocessing - except ImportError: - raise MultiprocessingUnsupported - if self.queue is not None: - raise RuntimeError("already doing multiple processes") - self.queue = multiprocessing.JoinableQueue() - self.output_lock = multiprocessing.Lock() - processes = [multiprocessing.Process(target=self._child) - for i in range(num_processes)] - try: - for p in processes: - p.start() - super(MultiprocessRefactoringTool, self).refactor(items, write, - doctests_only) - finally: - self.queue.join() - for i in range(num_processes): - self.queue.put(None) - for p in processes: - if p.is_alive(): - p.join() - self.queue = None - - def _child(self): - task = self.queue.get() - while task is not None: - args, kwargs = task - try: - super(MultiprocessRefactoringTool, self).refactor_file( - *args, **kwargs) - finally: - self.queue.task_done() - task = self.queue.get() - - def refactor_file(self, *args, **kwargs): - if self.queue is not None: - self.queue.put((args, kwargs)) - else: - return super(MultiprocessRefactoringTool, self).refactor_file( - *args, **kwargs) diff --git a/Lib/linecache.py b/Lib/linecache.py index 97644a8e3794e1..329a14053458b7 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,10 +5,8 @@ that name. """ -import functools import sys import os -import tokenize __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -82,6 +80,8 @@ def updatecache(filename, module_globals=None): If something's wrong, print a message, discard the cache entry, and return an empty list.""" + import tokenize + if filename in cache: if len(cache[filename]) != 1: cache.pop(filename, None) @@ -176,7 +176,16 @@ def lazycache(filename, module_globals): get_source = getattr(loader, 'get_source', None) if name and get_source: - get_lines = functools.partial(get_source, name) + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) cache[filename] = (get_lines,) return True return False + + +def _register_code(code, string, name): + cache[code] = ( + len(string), + None, + [line + '\n' for line in string.splitlines()], + name) diff --git a/Lib/locale.py b/Lib/locale.py index e94f0d1acbaa7d..e0cb4c5449d556 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -25,7 +25,7 @@ # Yuck: LC_MESSAGES is non-standard: can't tell whether it exists before # trying the import. So __all__ is also fiddled at the end of the file. __all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error", - "setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm", + "setlocale", "localeconv", "strcoll", "strxfrm", "str", "atof", "atoi", "format_string", "currency", "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY", "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding"] @@ -541,12 +541,14 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): """ import warnings - warnings.warn( - "Use setlocale(), getencoding() and getlocale() instead", - DeprecationWarning, stacklevel=2 - ) + warnings._deprecated( + "locale.getdefaultlocale", + "{name!r} is deprecated and slated for removal in Python {remove}. " + "Use setlocale(), getencoding() and getlocale() instead.", + remove=(3, 15)) return _getdefaultlocale(envvars) + def _getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): try: # check if it's supported by the _locale module @@ -612,40 +614,15 @@ def setlocale(category, locale=None): locale = normalize(_build_localename(locale)) return _setlocale(category, locale) -def resetlocale(category=LC_ALL): - - """ Sets the locale for category to the default setting. - - The default setting is determined by calling - getdefaultlocale(). category defaults to LC_ALL. - - """ - import warnings - warnings.warn( - 'Use locale.setlocale(locale.LC_ALL, "") instead', - DeprecationWarning, stacklevel=2 - ) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - loc = getdefaultlocale() - - _setlocale(category, _build_localename(loc)) - try: from _locale import getencoding except ImportError: + # When _locale.getencoding() is missing, locale.getencoding() uses the + # Python filesystem encoding. def getencoding(): - if hasattr(sys, 'getandroidapilevel'): - # On Android langinfo.h and CODESET are missing, and UTF-8 is - # always used in mbstowcs() and wcstombs(). - return 'utf-8' - encoding = _getdefaultlocale()[1] - if encoding is None: - # LANG not set, default to UTF-8 - encoding = 'utf-8' - return encoding + return sys.getfilesystemencoding() + try: CODESET @@ -1729,17 +1706,6 @@ def _init_categories(categories=categories): print(' Encoding: ', enc or '(undefined)') print() - print() - print('Locale settings after calling resetlocale():') - print('-'*72) - resetlocale() - for name,category in categories.items(): - print(name, '...') - lang, enc = getlocale(category) - print(' Language: ', lang or '(undefined)') - print(' Encoding: ', enc or '(undefined)') - print() - try: setlocale(LC_ALL, "") except: diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 056380fb2287af..eb7e020d1edfc0 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -37,7 +37,7 @@ 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', - 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', + 'warning', 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort', 'raiseExceptions', 'getLevelNamesMapping', 'getHandlerByName', 'getHandlerNames'] @@ -159,12 +159,9 @@ def addLevelName(level, levelName): This is used when converting levels to text during message formatting. """ - _acquireLock() - try: #unlikely to cause an exception, but you never know... + with _lock: _levelToName[level] = levelName _nameToLevel[levelName] = level - finally: - _releaseLock() if hasattr(sys, "_getframe"): currentframe = lambda: sys._getframe(1) @@ -231,21 +228,27 @@ def _checkLevel(level): # _lock = threading.RLock() -def _acquireLock(): +def _prepareFork(): """ - Acquire the module-level lock for serializing access to shared data. + Prepare to fork a new child process by acquiring the module-level lock. - This should be released with _releaseLock(). + This should be used in conjunction with _afterFork(). """ - if _lock: + # Wrap the lock acquisition in a try-except to prevent the lock from being + # abandoned in the event of an asynchronous exception. See gh-106238. + try: _lock.acquire() + except BaseException: + _lock.release() + raise -def _releaseLock(): +def _afterFork(): """ - Release the module-level lock acquired by calling _acquireLock(). + After a new child process has been forked, release the module-level lock. + + This should be used in conjunction with _prepareFork(). """ - if _lock: - _lock.release() + _lock.release() # Prevent a held logging lock from blocking a child from logging. @@ -260,23 +263,20 @@ def _register_at_fork_reinit_lock(instance): _at_fork_reinit_lock_weakset = weakref.WeakSet() def _register_at_fork_reinit_lock(instance): - _acquireLock() - try: + with _lock: _at_fork_reinit_lock_weakset.add(instance) - finally: - _releaseLock() def _after_at_fork_child_reinit_locks(): for handler in _at_fork_reinit_lock_weakset: handler._at_fork_reinit() - # _acquireLock() was called in the parent before forking. + # _prepareFork() was called in the parent before forking. # The lock is reinitialized to unlocked state. _lock._at_fork_reinit() - os.register_at_fork(before=_acquireLock, + os.register_at_fork(before=_prepareFork, after_in_child=_after_at_fork_child_reinit_locks, - after_in_parent=_releaseLock) + after_in_parent=_afterFork) #--------------------------------------------------------------------------- @@ -658,7 +658,7 @@ def formatException(self, ei): # See issues #9427, #1553375. Commented out for now. #if getattr(self, 'fullstack', False): # traceback.print_stack(tb.tb_frame.f_back, file=sio) - traceback.print_exception(ei[0], ei[1], tb, None, sio) + traceback.print_exception(ei[0], ei[1], tb, limit=None, file=sio) s = sio.getvalue() sio.close() if s[-1:] == "\n": @@ -879,25 +879,20 @@ def _removeHandlerRef(wr): # set to None. It can also be called from another thread. So we need to # pre-emptively grab the necessary globals and check if they're None, # to prevent race conditions and failures during interpreter shutdown. - acquire, release, handlers = _acquireLock, _releaseLock, _handlerList - if acquire and release and handlers: - acquire() - try: - handlers.remove(wr) - except ValueError: - pass - finally: - release() + handlers, lock = _handlerList, _lock + if lock and handlers: + with lock: + try: + handlers.remove(wr) + except ValueError: + pass def _addHandlerRef(handler): """ Add a handler to the internal cleanup list using a weak reference. """ - _acquireLock() - try: + with _lock: _handlerList.append(weakref.ref(handler, _removeHandlerRef)) - finally: - _releaseLock() def getHandlerByName(name): @@ -912,8 +907,7 @@ def getHandlerNames(): """ Return all known handler names as an immutable set. """ - result = set(_handlers.keys()) - return frozenset(result) + return frozenset(_handlers) class Handler(Filterer): @@ -943,15 +937,12 @@ def get_name(self): return self._name def set_name(self, name): - _acquireLock() - try: + with _lock: if self._name in _handlers: del _handlers[self._name] self._name = name if name: _handlers[name] = self - finally: - _releaseLock() name = property(get_name, set_name) @@ -1023,11 +1014,8 @@ def handle(self, record): if isinstance(rv, LogRecord): record = rv if rv: - self.acquire() - try: + with self.lock: self.emit(record) - finally: - self.release() return rv def setFormatter(self, fmt): @@ -1055,13 +1043,10 @@ def close(self): methods. """ #get the module data lock, as we're updating a shared structure. - _acquireLock() - try: #unlikely to raise an exception, but you never know... + with _lock: self._closed = True if self._name and self._name in _handlers: del _handlers[self._name] - finally: - _releaseLock() def handleError(self, record): """ @@ -1076,14 +1061,14 @@ def handleError(self, record): The record which was being processed is passed in to this method. """ if raiseExceptions and sys.stderr: # see issue 13807 - t, v, tb = sys.exc_info() + exc = sys.exception() try: sys.stderr.write('--- Logging error ---\n') - traceback.print_exception(t, v, tb, None, sys.stderr) + traceback.print_exception(exc, limit=None, file=sys.stderr) sys.stderr.write('Call stack:\n') # Walk the stack frame up until we're out of logging, # so as to print the calling context. - frame = tb.tb_frame + frame = exc.__traceback__.tb_frame while (frame and os.path.dirname(frame.f_code.co_filename) == __path__[0]): frame = frame.f_back @@ -1108,7 +1093,7 @@ def handleError(self, record): except OSError: #pragma: no cover pass # see issue 5971 finally: - del t, v, tb + del exc def __repr__(self): level = getLevelName(self.level) @@ -1138,12 +1123,9 @@ def flush(self): """ Flushes the stream. """ - self.acquire() - try: + with self.lock: if self.stream and hasattr(self.stream, "flush"): self.stream.flush() - finally: - self.release() def emit(self, record): """ @@ -1179,12 +1161,9 @@ def setStream(self, stream): result = None else: result = self.stream - self.acquire() - try: + with self.lock: self.flush() self.stream = stream - finally: - self.release() return result def __repr__(self): @@ -1234,8 +1213,7 @@ def close(self): """ Closes the stream. """ - self.acquire() - try: + with self.lock: try: if self.stream: try: @@ -1251,8 +1229,6 @@ def close(self): # Also see Issue #42378: we also rely on # self._closed being set to True there StreamHandler.close(self) - finally: - self.release() def _open(self): """ @@ -1388,8 +1364,7 @@ def getLogger(self, name): rv = None if not isinstance(name, str): raise TypeError('A logger name must be a string') - _acquireLock() - try: + with _lock: if name in self.loggerDict: rv = self.loggerDict[name] if isinstance(rv, PlaceHolder): @@ -1404,8 +1379,6 @@ def getLogger(self, name): rv.manager = self self.loggerDict[name] = rv self._fixupParents(rv) - finally: - _releaseLock() return rv def setLoggerClass(self, klass): @@ -1468,12 +1441,11 @@ def _clear_cache(self): Called when level changes are made """ - _acquireLock() - for logger in self.loggerDict.values(): - if isinstance(logger, Logger): - logger._cache.clear() - self.root._cache.clear() - _releaseLock() + with _lock: + for logger in self.loggerDict.values(): + if isinstance(logger, Logger): + logger._cache.clear() + self.root._cache.clear() #--------------------------------------------------------------------------- # Logger classes and functions @@ -1550,11 +1522,6 @@ def warning(self, msg, *args, **kwargs): if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) - def warn(self, msg, *args, **kwargs): - warnings.warn("The 'warn' method is deprecated, " - "use 'warning' instead", DeprecationWarning, 2) - self.warning(msg, *args, **kwargs) - def error(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'ERROR'. @@ -1703,23 +1670,17 @@ def addHandler(self, hdlr): """ Add the specified handler to this logger. """ - _acquireLock() - try: + with _lock: if not (hdlr in self.handlers): self.handlers.append(hdlr) - finally: - _releaseLock() def removeHandler(self, hdlr): """ Remove the specified handler from this logger. """ - _acquireLock() - try: + with _lock: if hdlr in self.handlers: self.handlers.remove(hdlr) - finally: - _releaseLock() def hasHandlers(self): """ @@ -1797,16 +1758,13 @@ def isEnabledFor(self, level): try: return self._cache[level] except KeyError: - _acquireLock() - try: + with _lock: if self.manager.disable >= level: is_enabled = self._cache[level] = False else: is_enabled = self._cache[level] = ( level >= self.getEffectiveLevel() ) - finally: - _releaseLock() return is_enabled def getChild(self, suffix): @@ -1836,16 +1794,13 @@ def _hierlevel(logger): return 1 + logger.name.count('.') d = self.manager.loggerDict - _acquireLock() - try: + with _lock: # exclude PlaceHolders - the last check is to ensure that lower-level # descendants aren't returned - if there are placeholders, a logger's # parent field might point to a grandparent or ancestor thereof. return set(item for item in d.values() if isinstance(item, Logger) and item.parent is self and _hierlevel(item) == 1 + _hierlevel(item.parent)) - finally: - _releaseLock() def __repr__(self): level = getLevelName(self.getEffectiveLevel()) @@ -1881,7 +1836,7 @@ class LoggerAdapter(object): information in logging output. """ - def __init__(self, logger, extra=None): + def __init__(self, logger, extra=None, merge_extra=False): """ Initialize the adapter with a logger and a dict-like object which provides contextual information. This constructor signature allows @@ -1891,9 +1846,20 @@ def __init__(self, logger, extra=None): following example: adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2")) + + By default, LoggerAdapter objects will drop the "extra" argument + passed on the individual log calls to use its own instead. + + Initializing it with merge_extra=True will instead merge both + maps when logging, the individual call extra taking precedence + over the LoggerAdapter instance extra + + .. versionchanged:: 3.13 + The *merge_extra* argument was added. """ self.logger = logger self.extra = extra + self.merge_extra = merge_extra def process(self, msg, kwargs): """ @@ -1905,7 +1871,10 @@ def process(self, msg, kwargs): Normally, you'll only need to override this one method in a LoggerAdapter subclass for your specific needs. """ - kwargs["extra"] = self.extra + if self.merge_extra and "extra" in kwargs: + kwargs["extra"] = {**self.extra, **kwargs["extra"]} + else: + kwargs["extra"] = self.extra return msg, kwargs # @@ -1929,11 +1898,6 @@ def warning(self, msg, *args, **kwargs): """ self.log(WARNING, msg, *args, **kwargs) - def warn(self, msg, *args, **kwargs): - warnings.warn("The 'warn' method is deprecated, " - "use 'warning' instead", DeprecationWarning, 2) - self.warning(msg, *args, **kwargs) - def error(self, msg, *args, **kwargs): """ Delegate an error call to the underlying logger. @@ -2095,8 +2059,7 @@ def basicConfig(**kwargs): """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads - _acquireLock() - try: + with _lock: force = kwargs.pop('force', False) encoding = kwargs.pop('encoding', None) errors = kwargs.pop('errors', 'backslashreplace') @@ -2145,8 +2108,6 @@ def basicConfig(**kwargs): if kwargs: keys = ', '.join(kwargs.keys()) raise ValueError('Unrecognised argument(s): %s' % keys) - finally: - _releaseLock() #--------------------------------------------------------------------------- # Utility functions at module level. @@ -2207,11 +2168,6 @@ def warning(msg, *args, **kwargs): basicConfig() root.warning(msg, *args, **kwargs) -def warn(msg, *args, **kwargs): - warnings.warn("The 'warn' function is deprecated, " - "use 'warning' instead", DeprecationWarning, 2) - warning(msg, *args, **kwargs) - def info(msg, *args, **kwargs): """ Log a message with severity 'INFO' on the root logger. If the logger has diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 16c54a6a4f7a2f..4b520e3b1e0da6 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -29,6 +29,7 @@ import io import logging import logging.handlers +import os import queue import re import struct @@ -60,28 +61,34 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non """ import configparser + if isinstance(fname, str): + if not os.path.exists(fname): + raise FileNotFoundError(f"{fname} doesn't exist") + elif not os.path.getsize(fname): + raise RuntimeError(f'{fname} is an empty file') + if isinstance(fname, configparser.RawConfigParser): cp = fname else: - cp = configparser.ConfigParser(defaults) - if hasattr(fname, 'readline'): - cp.read_file(fname) - else: - encoding = io.text_encoding(encoding) - cp.read(fname, encoding=encoding) + try: + cp = configparser.ConfigParser(defaults) + if hasattr(fname, 'readline'): + cp.read_file(fname) + else: + encoding = io.text_encoding(encoding) + cp.read(fname, encoding=encoding) + except configparser.ParsingError as e: + raise RuntimeError(f'{fname} is invalid: {e}') formatters = _create_formatters(cp) # critical section - logging._acquireLock() - try: + with logging._lock: _clearExistingHandlers() # Handlers add themselves to logging._handlers handlers = _install_handlers(cp, formatters) _install_loggers(cp, handlers, disable_existing_loggers) - finally: - logging._releaseLock() def _resolve(name): @@ -368,7 +375,7 @@ class BaseConfigurator(object): WORD_PATTERN = re.compile(r'^\s*(\w+)\s*') DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*') - INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*') + INDEX_PATTERN = re.compile(r'^\[([^\[\]]*)\]\s*') DIGIT_PATTERN = re.compile(r'^\d+$') value_converters = { @@ -475,10 +482,10 @@ def configure_custom(self, config): c = config.pop('()') if not callable(c): c = self.resolve(c) - props = config.pop('.', None) # Check for valid identifiers - kwargs = {k: config[k] for k in config if valid_ident(k)} + kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))} result = c(**kwargs) + props = config.pop('.', None) if props: for name, value in props.items(): setattr(result, name, value) @@ -506,8 +513,7 @@ def configure(self): raise ValueError("Unsupported version: %s" % config['version']) incremental = config.pop('incremental', False) EMPTY_DICT = {} - logging._acquireLock() - try: + with logging._lock: if incremental: handlers = config.get('handlers', EMPTY_DICT) for name in handlers: @@ -651,8 +657,6 @@ def configure(self): except Exception as e: raise ValueError('Unable to configure root ' 'logger') from e - finally: - logging._releaseLock() def configure_formatter(self, config): """Configure a formatter from a dictionary.""" @@ -831,8 +835,7 @@ def configure_handler(self, config): factory = functools.partial(self._configure_queue_handler, klass) else: factory = klass - props = config.pop('.', None) - kwargs = {k: config[k] for k in config if valid_ident(k)} + kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))} try: result = factory(**kwargs) except TypeError as te: @@ -850,6 +853,7 @@ def configure_handler(self, config): result.setLevel(logging._checkLevel(level)) if filters: self.add_filters(result, filters) + props = config.pop('.', None) if props: for name, value in props.items(): setattr(result, name, value) @@ -978,9 +982,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): ThreadingTCPServer.__init__(self, (host, port), handler) - logging._acquireLock() - self.abort = 0 - logging._releaseLock() + with logging._lock: + self.abort = 0 self.timeout = 1 self.ready = ready self.verify = verify @@ -994,9 +997,8 @@ def serve_until_stopped(self): self.timeout) if rd: self.handle_request() - logging._acquireLock() - abort = self.abort - logging._releaseLock() + with logging._lock: + abort = self.abort self.server_close() class Server(threading.Thread): @@ -1017,9 +1019,8 @@ def run(self): self.port = server.server_address[1] self.ready.set() global _listener - logging._acquireLock() - _listener = server - logging._releaseLock() + with logging._lock: + _listener = server server.serve_until_stopped() return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify) @@ -1029,10 +1030,7 @@ def stopListening(): Stop the listening server which was created with a call to listen(). """ global _listener - logging._acquireLock() - try: + with logging._lock: if _listener: _listener.abort = 1 _listener = None - finally: - logging._releaseLock() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 9847104446eaf6..9840b7b0aeba88 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -683,15 +683,12 @@ def close(self): """ Closes the socket. """ - self.acquire() - try: + with self.lock: sock = self.sock if sock: self.sock = None sock.close() logging.Handler.close(self) - finally: - self.release() class DatagramHandler(SocketHandler): """ @@ -833,10 +830,8 @@ class SysLogHandler(logging.Handler): "local7": LOG_LOCAL7, } - #The map below appears to be trivially lowercasing the key. However, - #there's more to it than meets the eye - in some locales, lowercasing - #gives unexpected results. See SF #1524081: in the Turkish locale, - #"INFO".lower() != "info" + # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept + # for backwards compatibility. priority_map = { "DEBUG" : "debug", "INFO" : "info", @@ -953,15 +948,12 @@ def close(self): """ Closes the socket. """ - self.acquire() - try: + with self.lock: sock = self.socket if sock: self.socket = None sock.close() logging.Handler.close(self) - finally: - self.release() def mapPriority(self, levelName): """ @@ -1333,11 +1325,8 @@ def flush(self): This version just zaps the buffer to empty. """ - self.acquire() - try: + with self.lock: self.buffer.clear() - finally: - self.release() def close(self): """ @@ -1387,11 +1376,8 @@ def setTarget(self, target): """ Set the target handler for this handler. """ - self.acquire() - try: + with self.lock: self.target = target - finally: - self.release() def flush(self): """ @@ -1399,16 +1385,13 @@ def flush(self): records to the target, if there is one. Override if you want different behaviour. - The record buffer is also cleared by this operation. + The record buffer is only cleared if a target has been set. """ - self.acquire() - try: + with self.lock: if self.target: for record in self.buffer: self.target.handle(record) self.buffer.clear() - finally: - self.release() def close(self): """ @@ -1419,12 +1402,9 @@ def close(self): if self.flushOnClose: self.flush() finally: - self.acquire() - try: + with self.lock: self.target = None BufferingHandler.close(self) - finally: - self.release() class QueueHandler(logging.Handler): diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 59834a2b3b5243..36afaded705d0a 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -395,6 +395,56 @@ def get_file(self, key): f = open(os.path.join(self._path, self._lookup(key)), 'rb') return _ProxyFile(f) + def get_info(self, key): + """Get the keyed message's "info" as a string.""" + subpath = self._lookup(key) + if self.colon in subpath: + return subpath.split(self.colon)[-1] + return '' + + def set_info(self, key, info: str): + """Set the keyed message's "info" string.""" + if not isinstance(info, str): + raise TypeError(f'info must be a string: {type(info)}') + old_subpath = self._lookup(key) + new_subpath = old_subpath.split(self.colon)[0] + if info: + new_subpath += self.colon + info + if new_subpath == old_subpath: + return + old_path = os.path.join(self._path, old_subpath) + new_path = os.path.join(self._path, new_subpath) + os.rename(old_path, new_path) + self._toc[key] = new_subpath + + def get_flags(self, key): + """Return as a string the standard flags that are set on the keyed message.""" + info = self.get_info(key) + if info.startswith('2,'): + return info[2:] + return '' + + def set_flags(self, key, flags: str): + """Set the given flags and unset all others on the keyed message.""" + if not isinstance(flags, str): + raise TypeError(f'flags must be a string: {type(flags)}') + # TODO: check if flags are valid standard flag characters? + self.set_info(key, '2,' + ''.join(sorted(set(flags)))) + + def add_flag(self, key, flag: str): + """Set the given flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + # TODO: check that flag is a valid standard flag character? + self.set_flags(key, ''.join(set(self.get_flags(key)) | set(flag))) + + def remove_flag(self, key, flag: str): + """Unset the given string flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + if self.get_flags(key): + self.set_flags(key, ''.join(set(self.get_flags(key)) - set(flag))) + def iterkeys(self): """Return an iterator over keys.""" self._refresh() diff --git a/Lib/mailcap.py b/Lib/mailcap.py deleted file mode 100644 index 2f4656e854b3bb..00000000000000 --- a/Lib/mailcap.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Mailcap file handling. See RFC 1524.""" - -import os -import warnings -import re - -__all__ = ["getcaps","findmatch"] - - -_DEPRECATION_MSG = ('The {name} module is deprecated and will be removed in ' - 'Python {remove}. See the mimetypes module for an ' - 'alternative.') -warnings._deprecated(__name__, _DEPRECATION_MSG, remove=(3, 13)) - - -def lineno_sort_key(entry): - # Sort in ascending order, with unspecified entries at the end - if 'lineno' in entry: - return 0, entry['lineno'] - else: - return 1, 0 - -_find_unsafe = re.compile(r'[^\xa1-\U0010FFFF\w@+=:,./-]').search - -class UnsafeMailcapInput(Warning): - """Warning raised when refusing unsafe input""" - - -# Part 1: top-level interface. - -def getcaps(): - """Return a dictionary containing the mailcap database. - - The dictionary maps a MIME type (in all lowercase, e.g. 'text/plain') - to a list of dictionaries corresponding to mailcap entries. The list - collects all the entries for that MIME type from all available mailcap - files. Each dictionary contains key-value pairs for that MIME type, - where the viewing command is stored with the key "view". - - """ - caps = {} - lineno = 0 - for mailcap in listmailcapfiles(): - try: - fp = open(mailcap, 'r') - except OSError: - continue - with fp: - morecaps, lineno = _readmailcapfile(fp, lineno) - for key, value in morecaps.items(): - if not key in caps: - caps[key] = value - else: - caps[key] = caps[key] + value - return caps - -def listmailcapfiles(): - """Return a list of all mailcap files found on the system.""" - # This is mostly a Unix thing, but we use the OS path separator anyway - if 'MAILCAPS' in os.environ: - pathstr = os.environ['MAILCAPS'] - mailcaps = pathstr.split(os.pathsep) - else: - if 'HOME' in os.environ: - home = os.environ['HOME'] - else: - # Don't bother with getpwuid() - home = '.' # Last resort - mailcaps = [home + '/.mailcap', '/etc/mailcap', - '/usr/etc/mailcap', '/usr/local/etc/mailcap'] - return mailcaps - - -# Part 2: the parser. -def readmailcapfile(fp): - """Read a mailcap file and return a dictionary keyed by MIME type.""" - warnings.warn('readmailcapfile is deprecated, use getcaps instead', - DeprecationWarning, 2) - caps, _ = _readmailcapfile(fp, None) - return caps - - -def _readmailcapfile(fp, lineno): - """Read a mailcap file and return a dictionary keyed by MIME type. - - Each MIME type is mapped to an entry consisting of a list of - dictionaries; the list will contain more than one such dictionary - if a given MIME type appears more than once in the mailcap file. - Each dictionary contains key-value pairs for that MIME type, where - the viewing command is stored with the key "view". - """ - caps = {} - while line := fp.readline(): - # Ignore comments and blank lines - if line[0] == '#' or line.strip() == '': - continue - nextline = line - # Join continuation lines - while nextline[-2:] == '\\\n': - nextline = fp.readline() - if not nextline: nextline = '\n' - line = line[:-2] + nextline - # Parse the line - key, fields = parseline(line) - if not (key and fields): - continue - if lineno is not None: - fields['lineno'] = lineno - lineno += 1 - # Normalize the key - types = key.split('/') - for j in range(len(types)): - types[j] = types[j].strip() - key = '/'.join(types).lower() - # Update the database - if key in caps: - caps[key].append(fields) - else: - caps[key] = [fields] - return caps, lineno - -def parseline(line): - """Parse one entry in a mailcap file and return a dictionary. - - The viewing command is stored as the value with the key "view", - and the rest of the fields produce key-value pairs in the dict. - """ - fields = [] - i, n = 0, len(line) - while i < n: - field, i = parsefield(line, i, n) - fields.append(field) - i = i+1 # Skip semicolon - if len(fields) < 2: - return None, None - key, view, rest = fields[0], fields[1], fields[2:] - fields = {'view': view} - for field in rest: - i = field.find('=') - if i < 0: - fkey = field - fvalue = "" - else: - fkey = field[:i].strip() - fvalue = field[i+1:].strip() - if fkey in fields: - # Ignore it - pass - else: - fields[fkey] = fvalue - return key, fields - -def parsefield(line, i, n): - """Separate one key-value pair in a mailcap entry.""" - start = i - while i < n: - c = line[i] - if c == ';': - break - elif c == '\\': - i = i+2 - else: - i = i+1 - return line[start:i].strip(), i - - -# Part 3: using the database. - -def findmatch(caps, MIMEtype, key='view', filename="/dev/null", plist=[]): - """Find a match for a mailcap entry. - - Return a tuple containing the command line, and the mailcap entry - used; (None, None) if no match is found. This may invoke the - 'test' command of several matching entries before deciding which - entry to use. - - """ - if _find_unsafe(filename): - msg = "Refusing to use mailcap with filename %r. Use a safe temporary filename." % (filename,) - warnings.warn(msg, UnsafeMailcapInput) - return None, None - entries = lookup(caps, MIMEtype, key) - # XXX This code should somehow check for the needsterminal flag. - for e in entries: - if 'test' in e: - test = subst(e['test'], filename, plist) - if test is None: - continue - if test and os.system(test) != 0: - continue - command = subst(e[key], MIMEtype, filename, plist) - if command is not None: - return command, e - return None, None - -def lookup(caps, MIMEtype, key=None): - entries = [] - if MIMEtype in caps: - entries = entries + caps[MIMEtype] - MIMEtypes = MIMEtype.split('/') - MIMEtype = MIMEtypes[0] + '/*' - if MIMEtype in caps: - entries = entries + caps[MIMEtype] - if key is not None: - entries = [e for e in entries if key in e] - entries = sorted(entries, key=lineno_sort_key) - return entries - -def subst(field, MIMEtype, filename, plist=[]): - # XXX Actually, this is Unix-specific - res = '' - i, n = 0, len(field) - while i < n: - c = field[i]; i = i+1 - if c != '%': - if c == '\\': - c = field[i:i+1]; i = i+1 - res = res + c - else: - c = field[i]; i = i+1 - if c == '%': - res = res + c - elif c == 's': - res = res + filename - elif c == 't': - if _find_unsafe(MIMEtype): - msg = "Refusing to substitute MIME type %r into a shell command." % (MIMEtype,) - warnings.warn(msg, UnsafeMailcapInput) - return None - res = res + MIMEtype - elif c == '{': - start = i - while i < n and field[i] != '}': - i = i+1 - name = field[start:i] - i = i+1 - param = findparam(name, plist) - if _find_unsafe(param): - msg = "Refusing to substitute parameter %r (%s) into a shell command" % (param, name) - warnings.warn(msg, UnsafeMailcapInput) - return None - res = res + param - # XXX To do: - # %n == number of parts if type is multipart/* - # %F == list of alternating type and filename for parts - else: - res = res + '%' + c - return res - -def findparam(name, plist): - name = name.lower() + '=' - n = len(name) - for p in plist: - if p[:n].lower() == name: - return p[n:] - return '' - - -# Part 4: test program. - -def test(): - import sys - caps = getcaps() - if not sys.argv[1:]: - show(caps) - return - for i in range(1, len(sys.argv), 2): - args = sys.argv[i:i+2] - if len(args) < 2: - print("usage: mailcap [MIMEtype file] ...") - return - MIMEtype = args[0] - file = args[1] - command, e = findmatch(caps, MIMEtype, 'view', file) - if not command: - print("No viewer found for", type) - else: - print("Executing:", command) - sts = os.system(command) - sts = os.waitstatus_to_exitcode(sts) - if sts: - print("Exit status:", sts) - -def show(caps): - print("Mailcap files:") - for fn in listmailcapfiles(): print("\t" + fn) - print() - if not caps: caps = getcaps() - print("Mailcap entries:") - print() - ckeys = sorted(caps) - for type in ckeys: - print(type) - entries = caps[type] - for e in entries: - keys = sorted(e) - for k in keys: - print(" %-15s" % k, e[k]) - print() - -if __name__ == '__main__': - test() diff --git a/Lib/msilib/__init__.py b/Lib/msilib/__init__.py deleted file mode 100644 index 565bf631abd4ed..00000000000000 --- a/Lib/msilib/__init__.py +++ /dev/null @@ -1,484 +0,0 @@ -# Copyright (C) 2005 Martin v. Löwis -# Licensed to PSF under a Contributor Agreement. -from _msi import * -import fnmatch -import os -import re -import string -import sys -import warnings - -warnings._deprecated(__name__, remove=(3, 13)) - -AMD64 = "AMD64" in sys.version -# Keep msilib.Win64 around to preserve backwards compatibility. -Win64 = AMD64 - -# Partially taken from Wine -datasizemask= 0x00ff -type_valid= 0x0100 -type_localizable= 0x0200 - -typemask= 0x0c00 -type_long= 0x0000 -type_short= 0x0400 -type_string= 0x0c00 -type_binary= 0x0800 - -type_nullable= 0x1000 -type_key= 0x2000 -# XXX temporary, localizable? -knownbits = datasizemask | type_valid | type_localizable | \ - typemask | type_nullable | type_key - -class Table: - def __init__(self, name): - self.name = name - self.fields = [] - - def add_field(self, index, name, type): - self.fields.append((index,name,type)) - - def sql(self): - fields = [] - keys = [] - self.fields.sort() - fields = [None]*len(self.fields) - for index, name, type in self.fields: - index -= 1 - unk = type & ~knownbits - if unk: - print("%s.%s unknown bits %x" % (self.name, name, unk)) - size = type & datasizemask - dtype = type & typemask - if dtype == type_string: - if size: - tname="CHAR(%d)" % size - else: - tname="CHAR" - elif dtype == type_short: - assert size==2 - tname = "SHORT" - elif dtype == type_long: - assert size==4 - tname="LONG" - elif dtype == type_binary: - assert size==0 - tname="OBJECT" - else: - tname="unknown" - print("%s.%sunknown integer type %d" % (self.name, name, size)) - if type & type_nullable: - flags = "" - else: - flags = " NOT NULL" - if type & type_localizable: - flags += " LOCALIZABLE" - fields[index] = "`%s` %s%s" % (name, tname, flags) - if type & type_key: - keys.append("`%s`" % name) - fields = ", ".join(fields) - keys = ", ".join(keys) - return "CREATE TABLE %s (%s PRIMARY KEY %s)" % (self.name, fields, keys) - - def create(self, db): - v = db.OpenView(self.sql()) - v.Execute(None) - v.Close() - -class _Unspecified:pass -def change_sequence(seq, action, seqno=_Unspecified, cond = _Unspecified): - "Change the sequence number of an action in a sequence list" - for i in range(len(seq)): - if seq[i][0] == action: - if cond is _Unspecified: - cond = seq[i][1] - if seqno is _Unspecified: - seqno = seq[i][2] - seq[i] = (action, cond, seqno) - return - raise ValueError("Action not found in sequence") - -def add_data(db, table, values): - v = db.OpenView("SELECT * FROM `%s`" % table) - count = v.GetColumnInfo(MSICOLINFO_NAMES).GetFieldCount() - r = CreateRecord(count) - for value in values: - assert len(value) == count, value - for i in range(count): - field = value[i] - if isinstance(field, int): - r.SetInteger(i+1,field) - elif isinstance(field, str): - r.SetString(i+1,field) - elif field is None: - pass - elif isinstance(field, Binary): - r.SetStream(i+1, field.name) - else: - raise TypeError("Unsupported type %s" % field.__class__.__name__) - try: - v.Modify(MSIMODIFY_INSERT, r) - except Exception: - raise MSIError("Could not insert "+repr(values)+" into "+table) - - r.ClearData() - v.Close() - - -def add_stream(db, name, path): - v = db.OpenView("INSERT INTO _Streams (Name, Data) VALUES ('%s', ?)" % name) - r = CreateRecord(1) - r.SetStream(1, path) - v.Execute(r) - v.Close() - -def init_database(name, schema, - ProductName, ProductCode, ProductVersion, - Manufacturer): - try: - os.unlink(name) - except OSError: - pass - ProductCode = ProductCode.upper() - # Create the database - db = OpenDatabase(name, MSIDBOPEN_CREATE) - # Create the tables - for t in schema.tables: - t.create(db) - # Fill the validation table - add_data(db, "_Validation", schema._Validation_records) - # Initialize the summary information, allowing atmost 20 properties - si = db.GetSummaryInformation(20) - si.SetProperty(PID_TITLE, "Installation Database") - si.SetProperty(PID_SUBJECT, ProductName) - si.SetProperty(PID_AUTHOR, Manufacturer) - if AMD64: - si.SetProperty(PID_TEMPLATE, "x64;1033") - else: - si.SetProperty(PID_TEMPLATE, "Intel;1033") - si.SetProperty(PID_REVNUMBER, gen_uuid()) - si.SetProperty(PID_WORDCOUNT, 2) # long file names, compressed, original media - si.SetProperty(PID_PAGECOUNT, 200) - si.SetProperty(PID_APPNAME, "Python MSI Library") - # XXX more properties - si.Persist() - add_data(db, "Property", [ - ("ProductName", ProductName), - ("ProductCode", ProductCode), - ("ProductVersion", ProductVersion), - ("Manufacturer", Manufacturer), - ("ProductLanguage", "1033")]) - db.Commit() - return db - -def add_tables(db, module): - for table in module.tables: - add_data(db, table, getattr(module, table)) - -def make_id(str): - identifier_chars = string.ascii_letters + string.digits + "._" - str = "".join([c if c in identifier_chars else "_" for c in str]) - if str[0] in (string.digits + "."): - str = "_" + str - assert re.match("^[A-Za-z_][A-Za-z0-9_.]*$", str), "FILE"+str - return str - -def gen_uuid(): - return "{"+UuidCreate().upper()+"}" - -class CAB: - def __init__(self, name): - self.name = name - self.files = [] - self.filenames = set() - self.index = 0 - - def gen_id(self, file): - logical = _logical = make_id(file) - pos = 1 - while logical in self.filenames: - logical = "%s.%d" % (_logical, pos) - pos += 1 - self.filenames.add(logical) - return logical - - def append(self, full, file, logical): - if os.path.isdir(full): - return - if not logical: - logical = self.gen_id(file) - self.index += 1 - self.files.append((full, logical)) - return self.index, logical - - def commit(self, db): - from tempfile import mktemp - filename = mktemp() - FCICreate(filename, self.files) - add_data(db, "Media", - [(1, self.index, None, "#"+self.name, None, None)]) - add_stream(db, self.name, filename) - os.unlink(filename) - db.Commit() - -_directories = set() -class Directory: - def __init__(self, db, cab, basedir, physical, _logical, default, componentflags=None): - """Create a new directory in the Directory table. There is a current component - at each point in time for the directory, which is either explicitly created - through start_component, or implicitly when files are added for the first - time. Files are added into the current component, and into the cab file. - To create a directory, a base directory object needs to be specified (can be - None), the path to the physical directory, and a logical directory name. - Default specifies the DefaultDir slot in the directory table. componentflags - specifies the default flags that new components get.""" - index = 1 - _logical = make_id(_logical) - logical = _logical - while logical in _directories: - logical = "%s%d" % (_logical, index) - index += 1 - _directories.add(logical) - self.db = db - self.cab = cab - self.basedir = basedir - self.physical = physical - self.logical = logical - self.component = None - self.short_names = set() - self.ids = set() - self.keyfiles = {} - self.componentflags = componentflags - if basedir: - self.absolute = os.path.join(basedir.absolute, physical) - blogical = basedir.logical - else: - self.absolute = physical - blogical = None - add_data(db, "Directory", [(logical, blogical, default)]) - - def start_component(self, component = None, feature = None, flags = None, keyfile = None, uuid=None): - """Add an entry to the Component table, and make this component the current for this - directory. If no component name is given, the directory name is used. If no feature - is given, the current feature is used. If no flags are given, the directory's default - flags are used. If no keyfile is given, the KeyPath is left null in the Component - table.""" - if flags is None: - flags = self.componentflags - if uuid is None: - uuid = gen_uuid() - else: - uuid = uuid.upper() - if component is None: - component = self.logical - self.component = component - if AMD64: - flags |= 256 - if keyfile: - keyid = self.cab.gen_id(keyfile) - self.keyfiles[keyfile] = keyid - else: - keyid = None - add_data(self.db, "Component", - [(component, uuid, self.logical, flags, None, keyid)]) - if feature is None: - feature = current_feature - add_data(self.db, "FeatureComponents", - [(feature.id, component)]) - - def make_short(self, file): - oldfile = file - file = file.replace('+', '_') - file = ''.join(c for c in file if not c in r' "/\[]:;=,') - parts = file.split(".") - if len(parts) > 1: - prefix = "".join(parts[:-1]).upper() - suffix = parts[-1].upper() - if not prefix: - prefix = suffix - suffix = None - else: - prefix = file.upper() - suffix = None - if len(parts) < 3 and len(prefix) <= 8 and file == oldfile and ( - not suffix or len(suffix) <= 3): - if suffix: - file = prefix+"."+suffix - else: - file = prefix - else: - file = None - if file is None or file in self.short_names: - prefix = prefix[:6] - if suffix: - suffix = suffix[:3] - pos = 1 - while 1: - if suffix: - file = "%s~%d.%s" % (prefix, pos, suffix) - else: - file = "%s~%d" % (prefix, pos) - if file not in self.short_names: break - pos += 1 - assert pos < 10000 - if pos in (10, 100, 1000): - prefix = prefix[:-1] - self.short_names.add(file) - assert not re.search(r'[\?|><:/*"+,;=\[\]]', file) # restrictions on short names - return file - - def add_file(self, file, src=None, version=None, language=None): - """Add a file to the current component of the directory, starting a new one - if there is no current component. By default, the file name in the source - and the file table will be identical. If the src file is specified, it is - interpreted relative to the current directory. Optionally, a version and a - language can be specified for the entry in the File table.""" - if not self.component: - self.start_component(self.logical, current_feature, 0) - if not src: - # Allow relative paths for file if src is not specified - src = file - file = os.path.basename(file) - absolute = os.path.join(self.absolute, src) - assert not re.search(r'[\?|><:/*]"', file) # restrictions on long names - if file in self.keyfiles: - logical = self.keyfiles[file] - else: - logical = None - sequence, logical = self.cab.append(absolute, file, logical) - assert logical not in self.ids - self.ids.add(logical) - short = self.make_short(file) - full = "%s|%s" % (short, file) - filesize = os.stat(absolute).st_size - # constants.msidbFileAttributesVital - # Compressed omitted, since it is the database default - # could add r/o, system, hidden - attributes = 512 - add_data(self.db, "File", - [(logical, self.component, full, filesize, version, - language, attributes, sequence)]) - #if not version: - # # Add hash if the file is not versioned - # filehash = FileHash(absolute, 0) - # add_data(self.db, "MsiFileHash", - # [(logical, 0, filehash.IntegerData(1), - # filehash.IntegerData(2), filehash.IntegerData(3), - # filehash.IntegerData(4))]) - # Automatically remove .pyc files on uninstall (2) - # XXX: adding so many RemoveFile entries makes installer unbelievably - # slow. So instead, we have to use wildcard remove entries - if file.endswith(".py"): - add_data(self.db, "RemoveFile", - [(logical+"c", self.component, "%sC|%sc" % (short, file), - self.logical, 2), - (logical+"o", self.component, "%sO|%so" % (short, file), - self.logical, 2)]) - return logical - - def glob(self, pattern, exclude = None): - """Add a list of files to the current component as specified in the - glob pattern. Individual files can be excluded in the exclude list.""" - try: - files = os.listdir(self.absolute) - except OSError: - return [] - if pattern[:1] != '.': - files = (f for f in files if f[0] != '.') - files = fnmatch.filter(files, pattern) - for f in files: - if exclude and f in exclude: continue - self.add_file(f) - return files - - def remove_pyc(self): - "Remove .pyc files on uninstall" - add_data(self.db, "RemoveFile", - [(self.component+"c", self.component, "*.pyc", self.logical, 2)]) - -class Binary: - def __init__(self, fname): - self.name = fname - def __repr__(self): - return 'msilib.Binary(os.path.join(dirname,"%s"))' % self.name - -class Feature: - def __init__(self, db, id, title, desc, display, level = 1, - parent=None, directory = None, attributes=0): - self.id = id - if parent: - parent = parent.id - add_data(db, "Feature", - [(id, parent, title, desc, display, - level, directory, attributes)]) - def set_current(self): - global current_feature - current_feature = self - -class Control: - def __init__(self, dlg, name): - self.dlg = dlg - self.name = name - - def event(self, event, argument, condition = "1", ordering = None): - add_data(self.dlg.db, "ControlEvent", - [(self.dlg.name, self.name, event, argument, - condition, ordering)]) - - def mapping(self, event, attribute): - add_data(self.dlg.db, "EventMapping", - [(self.dlg.name, self.name, event, attribute)]) - - def condition(self, action, condition): - add_data(self.dlg.db, "ControlCondition", - [(self.dlg.name, self.name, action, condition)]) - -class RadioButtonGroup(Control): - def __init__(self, dlg, name, property): - self.dlg = dlg - self.name = name - self.property = property - self.index = 1 - - def add(self, name, x, y, w, h, text, value = None): - if value is None: - value = name - add_data(self.dlg.db, "RadioButton", - [(self.property, self.index, value, - x, y, w, h, text, None)]) - self.index += 1 - -class Dialog: - def __init__(self, db, name, x, y, w, h, attr, title, first, default, cancel): - self.db = db - self.name = name - self.x, self.y, self.w, self.h = x,y,w,h - add_data(db, "Dialog", [(name, x,y,w,h,attr,title,first,default,cancel)]) - - def control(self, name, type, x, y, w, h, attr, prop, text, next, help): - add_data(self.db, "Control", - [(self.name, name, type, x, y, w, h, attr, prop, text, next, help)]) - return Control(self, name) - - def text(self, name, x, y, w, h, attr, text): - return self.control(name, "Text", x, y, w, h, attr, None, - text, None, None) - - def bitmap(self, name, x, y, w, h, text): - return self.control(name, "Bitmap", x, y, w, h, 1, None, text, None, None) - - def line(self, name, x, y, w, h): - return self.control(name, "Line", x, y, w, h, 1, None, None, None, None) - - def pushbutton(self, name, x, y, w, h, attr, text, next): - return self.control(name, "PushButton", x, y, w, h, attr, None, text, next, None) - - def radiogroup(self, name, x, y, w, h, attr, prop, text, next): - add_data(self.db, "Control", - [(self.name, name, "RadioButtonGroup", - x, y, w, h, attr, prop, text, next, None)]) - return RadioButtonGroup(self, name, prop) - - def checkbox(self, name, x, y, w, h, attr, prop, text, next): - return self.control(name, "CheckBox", x, y, w, h, attr, prop, text, next, None) diff --git a/Lib/msilib/schema.py b/Lib/msilib/schema.py deleted file mode 100644 index 9f5745cfb14de2..00000000000000 --- a/Lib/msilib/schema.py +++ /dev/null @@ -1,1007 +0,0 @@ -from . import Table - -_Validation = Table('_Validation') -_Validation.add_field(1,'Table',11552) -_Validation.add_field(2,'Column',11552) -_Validation.add_field(3,'Nullable',3332) -_Validation.add_field(4,'MinValue',4356) -_Validation.add_field(5,'MaxValue',4356) -_Validation.add_field(6,'KeyTable',7679) -_Validation.add_field(7,'KeyColumn',5378) -_Validation.add_field(8,'Category',7456) -_Validation.add_field(9,'Set',7679) -_Validation.add_field(10,'Description',7679) - -ActionText = Table('ActionText') -ActionText.add_field(1,'Action',11592) -ActionText.add_field(2,'Description',7936) -ActionText.add_field(3,'Template',7936) - -AdminExecuteSequence = Table('AdminExecuteSequence') -AdminExecuteSequence.add_field(1,'Action',11592) -AdminExecuteSequence.add_field(2,'Condition',7679) -AdminExecuteSequence.add_field(3,'Sequence',5378) - -Condition = Table('Condition') -Condition.add_field(1,'Feature_',11558) -Condition.add_field(2,'Level',9474) -Condition.add_field(3,'Condition',7679) - -AdminUISequence = Table('AdminUISequence') -AdminUISequence.add_field(1,'Action',11592) -AdminUISequence.add_field(2,'Condition',7679) -AdminUISequence.add_field(3,'Sequence',5378) - -AdvtExecuteSequence = Table('AdvtExecuteSequence') -AdvtExecuteSequence.add_field(1,'Action',11592) -AdvtExecuteSequence.add_field(2,'Condition',7679) -AdvtExecuteSequence.add_field(3,'Sequence',5378) - -AdvtUISequence = Table('AdvtUISequence') -AdvtUISequence.add_field(1,'Action',11592) -AdvtUISequence.add_field(2,'Condition',7679) -AdvtUISequence.add_field(3,'Sequence',5378) - -AppId = Table('AppId') -AppId.add_field(1,'AppId',11558) -AppId.add_field(2,'RemoteServerName',7679) -AppId.add_field(3,'LocalService',7679) -AppId.add_field(4,'ServiceParameters',7679) -AppId.add_field(5,'DllSurrogate',7679) -AppId.add_field(6,'ActivateAtStorage',5378) -AppId.add_field(7,'RunAsInteractiveUser',5378) - -AppSearch = Table('AppSearch') -AppSearch.add_field(1,'Property',11592) -AppSearch.add_field(2,'Signature_',11592) - -Property = Table('Property') -Property.add_field(1,'Property',11592) -Property.add_field(2,'Value',3840) - -BBControl = Table('BBControl') -BBControl.add_field(1,'Billboard_',11570) -BBControl.add_field(2,'BBControl',11570) -BBControl.add_field(3,'Type',3378) -BBControl.add_field(4,'X',1282) -BBControl.add_field(5,'Y',1282) -BBControl.add_field(6,'Width',1282) -BBControl.add_field(7,'Height',1282) -BBControl.add_field(8,'Attributes',4356) -BBControl.add_field(9,'Text',7986) - -Billboard = Table('Billboard') -Billboard.add_field(1,'Billboard',11570) -Billboard.add_field(2,'Feature_',3366) -Billboard.add_field(3,'Action',7474) -Billboard.add_field(4,'Ordering',5378) - -Feature = Table('Feature') -Feature.add_field(1,'Feature',11558) -Feature.add_field(2,'Feature_Parent',7462) -Feature.add_field(3,'Title',8000) -Feature.add_field(4,'Description',8191) -Feature.add_field(5,'Display',5378) -Feature.add_field(6,'Level',1282) -Feature.add_field(7,'Directory_',7496) -Feature.add_field(8,'Attributes',1282) - -Binary = Table('Binary') -Binary.add_field(1,'Name',11592) -Binary.add_field(2,'Data',2304) - -BindImage = Table('BindImage') -BindImage.add_field(1,'File_',11592) -BindImage.add_field(2,'Path',7679) - -File = Table('File') -File.add_field(1,'File',11592) -File.add_field(2,'Component_',3400) -File.add_field(3,'FileName',4095) -File.add_field(4,'FileSize',260) -File.add_field(5,'Version',7496) -File.add_field(6,'Language',7444) -File.add_field(7,'Attributes',5378) -File.add_field(8,'Sequence',1282) - -CCPSearch = Table('CCPSearch') -CCPSearch.add_field(1,'Signature_',11592) - -CheckBox = Table('CheckBox') -CheckBox.add_field(1,'Property',11592) -CheckBox.add_field(2,'Value',7488) - -Class = Table('Class') -Class.add_field(1,'CLSID',11558) -Class.add_field(2,'Context',11552) -Class.add_field(3,'Component_',11592) -Class.add_field(4,'ProgId_Default',7679) -Class.add_field(5,'Description',8191) -Class.add_field(6,'AppId_',7462) -Class.add_field(7,'FileTypeMask',7679) -Class.add_field(8,'Icon_',7496) -Class.add_field(9,'IconIndex',5378) -Class.add_field(10,'DefInprocHandler',7456) -Class.add_field(11,'Argument',7679) -Class.add_field(12,'Feature_',3366) -Class.add_field(13,'Attributes',5378) - -Component = Table('Component') -Component.add_field(1,'Component',11592) -Component.add_field(2,'ComponentId',7462) -Component.add_field(3,'Directory_',3400) -Component.add_field(4,'Attributes',1282) -Component.add_field(5,'Condition',7679) -Component.add_field(6,'KeyPath',7496) - -Icon = Table('Icon') -Icon.add_field(1,'Name',11592) -Icon.add_field(2,'Data',2304) - -ProgId = Table('ProgId') -ProgId.add_field(1,'ProgId',11775) -ProgId.add_field(2,'ProgId_Parent',7679) -ProgId.add_field(3,'Class_',7462) -ProgId.add_field(4,'Description',8191) -ProgId.add_field(5,'Icon_',7496) -ProgId.add_field(6,'IconIndex',5378) - -ComboBox = Table('ComboBox') -ComboBox.add_field(1,'Property',11592) -ComboBox.add_field(2,'Order',9474) -ComboBox.add_field(3,'Value',3392) -ComboBox.add_field(4,'Text',8000) - -CompLocator = Table('CompLocator') -CompLocator.add_field(1,'Signature_',11592) -CompLocator.add_field(2,'ComponentId',3366) -CompLocator.add_field(3,'Type',5378) - -Complus = Table('Complus') -Complus.add_field(1,'Component_',11592) -Complus.add_field(2,'ExpType',13570) - -Directory = Table('Directory') -Directory.add_field(1,'Directory',11592) -Directory.add_field(2,'Directory_Parent',7496) -Directory.add_field(3,'DefaultDir',4095) - -Control = Table('Control') -Control.add_field(1,'Dialog_',11592) -Control.add_field(2,'Control',11570) -Control.add_field(3,'Type',3348) -Control.add_field(4,'X',1282) -Control.add_field(5,'Y',1282) -Control.add_field(6,'Width',1282) -Control.add_field(7,'Height',1282) -Control.add_field(8,'Attributes',4356) -Control.add_field(9,'Property',7474) -Control.add_field(10,'Text',7936) -Control.add_field(11,'Control_Next',7474) -Control.add_field(12,'Help',7986) - -Dialog = Table('Dialog') -Dialog.add_field(1,'Dialog',11592) -Dialog.add_field(2,'HCentering',1282) -Dialog.add_field(3,'VCentering',1282) -Dialog.add_field(4,'Width',1282) -Dialog.add_field(5,'Height',1282) -Dialog.add_field(6,'Attributes',4356) -Dialog.add_field(7,'Title',8064) -Dialog.add_field(8,'Control_First',3378) -Dialog.add_field(9,'Control_Default',7474) -Dialog.add_field(10,'Control_Cancel',7474) - -ControlCondition = Table('ControlCondition') -ControlCondition.add_field(1,'Dialog_',11592) -ControlCondition.add_field(2,'Control_',11570) -ControlCondition.add_field(3,'Action',11570) -ControlCondition.add_field(4,'Condition',11775) - -ControlEvent = Table('ControlEvent') -ControlEvent.add_field(1,'Dialog_',11592) -ControlEvent.add_field(2,'Control_',11570) -ControlEvent.add_field(3,'Event',11570) -ControlEvent.add_field(4,'Argument',11775) -ControlEvent.add_field(5,'Condition',15871) -ControlEvent.add_field(6,'Ordering',5378) - -CreateFolder = Table('CreateFolder') -CreateFolder.add_field(1,'Directory_',11592) -CreateFolder.add_field(2,'Component_',11592) - -CustomAction = Table('CustomAction') -CustomAction.add_field(1,'Action',11592) -CustomAction.add_field(2,'Type',1282) -CustomAction.add_field(3,'Source',7496) -CustomAction.add_field(4,'Target',7679) - -DrLocator = Table('DrLocator') -DrLocator.add_field(1,'Signature_',11592) -DrLocator.add_field(2,'Parent',15688) -DrLocator.add_field(3,'Path',15871) -DrLocator.add_field(4,'Depth',5378) - -DuplicateFile = Table('DuplicateFile') -DuplicateFile.add_field(1,'FileKey',11592) -DuplicateFile.add_field(2,'Component_',3400) -DuplicateFile.add_field(3,'File_',3400) -DuplicateFile.add_field(4,'DestName',8191) -DuplicateFile.add_field(5,'DestFolder',7496) - -Environment = Table('Environment') -Environment.add_field(1,'Environment',11592) -Environment.add_field(2,'Name',4095) -Environment.add_field(3,'Value',8191) -Environment.add_field(4,'Component_',3400) - -Error = Table('Error') -Error.add_field(1,'Error',9474) -Error.add_field(2,'Message',7936) - -EventMapping = Table('EventMapping') -EventMapping.add_field(1,'Dialog_',11592) -EventMapping.add_field(2,'Control_',11570) -EventMapping.add_field(3,'Event',11570) -EventMapping.add_field(4,'Attribute',3378) - -Extension = Table('Extension') -Extension.add_field(1,'Extension',11775) -Extension.add_field(2,'Component_',11592) -Extension.add_field(3,'ProgId_',7679) -Extension.add_field(4,'MIME_',7488) -Extension.add_field(5,'Feature_',3366) - -MIME = Table('MIME') -MIME.add_field(1,'ContentType',11584) -MIME.add_field(2,'Extension_',3583) -MIME.add_field(3,'CLSID',7462) - -FeatureComponents = Table('FeatureComponents') -FeatureComponents.add_field(1,'Feature_',11558) -FeatureComponents.add_field(2,'Component_',11592) - -FileSFPCatalog = Table('FileSFPCatalog') -FileSFPCatalog.add_field(1,'File_',11592) -FileSFPCatalog.add_field(2,'SFPCatalog_',11775) - -SFPCatalog = Table('SFPCatalog') -SFPCatalog.add_field(1,'SFPCatalog',11775) -SFPCatalog.add_field(2,'Catalog',2304) -SFPCatalog.add_field(3,'Dependency',7424) - -Font = Table('Font') -Font.add_field(1,'File_',11592) -Font.add_field(2,'FontTitle',7552) - -IniFile = Table('IniFile') -IniFile.add_field(1,'IniFile',11592) -IniFile.add_field(2,'FileName',4095) -IniFile.add_field(3,'DirProperty',7496) -IniFile.add_field(4,'Section',3936) -IniFile.add_field(5,'Key',3968) -IniFile.add_field(6,'Value',4095) -IniFile.add_field(7,'Action',1282) -IniFile.add_field(8,'Component_',3400) - -IniLocator = Table('IniLocator') -IniLocator.add_field(1,'Signature_',11592) -IniLocator.add_field(2,'FileName',3583) -IniLocator.add_field(3,'Section',3424) -IniLocator.add_field(4,'Key',3456) -IniLocator.add_field(5,'Field',5378) -IniLocator.add_field(6,'Type',5378) - -InstallExecuteSequence = Table('InstallExecuteSequence') -InstallExecuteSequence.add_field(1,'Action',11592) -InstallExecuteSequence.add_field(2,'Condition',7679) -InstallExecuteSequence.add_field(3,'Sequence',5378) - -InstallUISequence = Table('InstallUISequence') -InstallUISequence.add_field(1,'Action',11592) -InstallUISequence.add_field(2,'Condition',7679) -InstallUISequence.add_field(3,'Sequence',5378) - -IsolatedComponent = Table('IsolatedComponent') -IsolatedComponent.add_field(1,'Component_Shared',11592) -IsolatedComponent.add_field(2,'Component_Application',11592) - -LaunchCondition = Table('LaunchCondition') -LaunchCondition.add_field(1,'Condition',11775) -LaunchCondition.add_field(2,'Description',4095) - -ListBox = Table('ListBox') -ListBox.add_field(1,'Property',11592) -ListBox.add_field(2,'Order',9474) -ListBox.add_field(3,'Value',3392) -ListBox.add_field(4,'Text',8000) - -ListView = Table('ListView') -ListView.add_field(1,'Property',11592) -ListView.add_field(2,'Order',9474) -ListView.add_field(3,'Value',3392) -ListView.add_field(4,'Text',8000) -ListView.add_field(5,'Binary_',7496) - -LockPermissions = Table('LockPermissions') -LockPermissions.add_field(1,'LockObject',11592) -LockPermissions.add_field(2,'Table',11552) -LockPermissions.add_field(3,'Domain',15871) -LockPermissions.add_field(4,'User',11775) -LockPermissions.add_field(5,'Permission',4356) - -Media = Table('Media') -Media.add_field(1,'DiskId',9474) -Media.add_field(2,'LastSequence',1282) -Media.add_field(3,'DiskPrompt',8000) -Media.add_field(4,'Cabinet',7679) -Media.add_field(5,'VolumeLabel',7456) -Media.add_field(6,'Source',7496) - -MoveFile = Table('MoveFile') -MoveFile.add_field(1,'FileKey',11592) -MoveFile.add_field(2,'Component_',3400) -MoveFile.add_field(3,'SourceName',8191) -MoveFile.add_field(4,'DestName',8191) -MoveFile.add_field(5,'SourceFolder',7496) -MoveFile.add_field(6,'DestFolder',3400) -MoveFile.add_field(7,'Options',1282) - -MsiAssembly = Table('MsiAssembly') -MsiAssembly.add_field(1,'Component_',11592) -MsiAssembly.add_field(2,'Feature_',3366) -MsiAssembly.add_field(3,'File_Manifest',7496) -MsiAssembly.add_field(4,'File_Application',7496) -MsiAssembly.add_field(5,'Attributes',5378) - -MsiAssemblyName = Table('MsiAssemblyName') -MsiAssemblyName.add_field(1,'Component_',11592) -MsiAssemblyName.add_field(2,'Name',11775) -MsiAssemblyName.add_field(3,'Value',3583) - -MsiDigitalCertificate = Table('MsiDigitalCertificate') -MsiDigitalCertificate.add_field(1,'DigitalCertificate',11592) -MsiDigitalCertificate.add_field(2,'CertData',2304) - -MsiDigitalSignature = Table('MsiDigitalSignature') -MsiDigitalSignature.add_field(1,'Table',11552) -MsiDigitalSignature.add_field(2,'SignObject',11592) -MsiDigitalSignature.add_field(3,'DigitalCertificate_',3400) -MsiDigitalSignature.add_field(4,'Hash',6400) - -MsiFileHash = Table('MsiFileHash') -MsiFileHash.add_field(1,'File_',11592) -MsiFileHash.add_field(2,'Options',1282) -MsiFileHash.add_field(3,'HashPart1',260) -MsiFileHash.add_field(4,'HashPart2',260) -MsiFileHash.add_field(5,'HashPart3',260) -MsiFileHash.add_field(6,'HashPart4',260) - -MsiPatchHeaders = Table('MsiPatchHeaders') -MsiPatchHeaders.add_field(1,'StreamRef',11558) -MsiPatchHeaders.add_field(2,'Header',2304) - -ODBCAttribute = Table('ODBCAttribute') -ODBCAttribute.add_field(1,'Driver_',11592) -ODBCAttribute.add_field(2,'Attribute',11560) -ODBCAttribute.add_field(3,'Value',8191) - -ODBCDriver = Table('ODBCDriver') -ODBCDriver.add_field(1,'Driver',11592) -ODBCDriver.add_field(2,'Component_',3400) -ODBCDriver.add_field(3,'Description',3583) -ODBCDriver.add_field(4,'File_',3400) -ODBCDriver.add_field(5,'File_Setup',7496) - -ODBCDataSource = Table('ODBCDataSource') -ODBCDataSource.add_field(1,'DataSource',11592) -ODBCDataSource.add_field(2,'Component_',3400) -ODBCDataSource.add_field(3,'Description',3583) -ODBCDataSource.add_field(4,'DriverDescription',3583) -ODBCDataSource.add_field(5,'Registration',1282) - -ODBCSourceAttribute = Table('ODBCSourceAttribute') -ODBCSourceAttribute.add_field(1,'DataSource_',11592) -ODBCSourceAttribute.add_field(2,'Attribute',11552) -ODBCSourceAttribute.add_field(3,'Value',8191) - -ODBCTranslator = Table('ODBCTranslator') -ODBCTranslator.add_field(1,'Translator',11592) -ODBCTranslator.add_field(2,'Component_',3400) -ODBCTranslator.add_field(3,'Description',3583) -ODBCTranslator.add_field(4,'File_',3400) -ODBCTranslator.add_field(5,'File_Setup',7496) - -Patch = Table('Patch') -Patch.add_field(1,'File_',11592) -Patch.add_field(2,'Sequence',9474) -Patch.add_field(3,'PatchSize',260) -Patch.add_field(4,'Attributes',1282) -Patch.add_field(5,'Header',6400) -Patch.add_field(6,'StreamRef_',7462) - -PatchPackage = Table('PatchPackage') -PatchPackage.add_field(1,'PatchId',11558) -PatchPackage.add_field(2,'Media_',1282) - -PublishComponent = Table('PublishComponent') -PublishComponent.add_field(1,'ComponentId',11558) -PublishComponent.add_field(2,'Qualifier',11775) -PublishComponent.add_field(3,'Component_',11592) -PublishComponent.add_field(4,'AppData',8191) -PublishComponent.add_field(5,'Feature_',3366) - -RadioButton = Table('RadioButton') -RadioButton.add_field(1,'Property',11592) -RadioButton.add_field(2,'Order',9474) -RadioButton.add_field(3,'Value',3392) -RadioButton.add_field(4,'X',1282) -RadioButton.add_field(5,'Y',1282) -RadioButton.add_field(6,'Width',1282) -RadioButton.add_field(7,'Height',1282) -RadioButton.add_field(8,'Text',8000) -RadioButton.add_field(9,'Help',7986) - -Registry = Table('Registry') -Registry.add_field(1,'Registry',11592) -Registry.add_field(2,'Root',1282) -Registry.add_field(3,'Key',4095) -Registry.add_field(4,'Name',8191) -Registry.add_field(5,'Value',7936) -Registry.add_field(6,'Component_',3400) - -RegLocator = Table('RegLocator') -RegLocator.add_field(1,'Signature_',11592) -RegLocator.add_field(2,'Root',1282) -RegLocator.add_field(3,'Key',3583) -RegLocator.add_field(4,'Name',7679) -RegLocator.add_field(5,'Type',5378) - -RemoveFile = Table('RemoveFile') -RemoveFile.add_field(1,'FileKey',11592) -RemoveFile.add_field(2,'Component_',3400) -RemoveFile.add_field(3,'FileName',8191) -RemoveFile.add_field(4,'DirProperty',3400) -RemoveFile.add_field(5,'InstallMode',1282) - -RemoveIniFile = Table('RemoveIniFile') -RemoveIniFile.add_field(1,'RemoveIniFile',11592) -RemoveIniFile.add_field(2,'FileName',4095) -RemoveIniFile.add_field(3,'DirProperty',7496) -RemoveIniFile.add_field(4,'Section',3936) -RemoveIniFile.add_field(5,'Key',3968) -RemoveIniFile.add_field(6,'Value',8191) -RemoveIniFile.add_field(7,'Action',1282) -RemoveIniFile.add_field(8,'Component_',3400) - -RemoveRegistry = Table('RemoveRegistry') -RemoveRegistry.add_field(1,'RemoveRegistry',11592) -RemoveRegistry.add_field(2,'Root',1282) -RemoveRegistry.add_field(3,'Key',4095) -RemoveRegistry.add_field(4,'Name',8191) -RemoveRegistry.add_field(5,'Component_',3400) - -ReserveCost = Table('ReserveCost') -ReserveCost.add_field(1,'ReserveKey',11592) -ReserveCost.add_field(2,'Component_',3400) -ReserveCost.add_field(3,'ReserveFolder',7496) -ReserveCost.add_field(4,'ReserveLocal',260) -ReserveCost.add_field(5,'ReserveSource',260) - -SelfReg = Table('SelfReg') -SelfReg.add_field(1,'File_',11592) -SelfReg.add_field(2,'Cost',5378) - -ServiceControl = Table('ServiceControl') -ServiceControl.add_field(1,'ServiceControl',11592) -ServiceControl.add_field(2,'Name',4095) -ServiceControl.add_field(3,'Event',1282) -ServiceControl.add_field(4,'Arguments',8191) -ServiceControl.add_field(5,'Wait',5378) -ServiceControl.add_field(6,'Component_',3400) - -ServiceInstall = Table('ServiceInstall') -ServiceInstall.add_field(1,'ServiceInstall',11592) -ServiceInstall.add_field(2,'Name',3583) -ServiceInstall.add_field(3,'DisplayName',8191) -ServiceInstall.add_field(4,'ServiceType',260) -ServiceInstall.add_field(5,'StartType',260) -ServiceInstall.add_field(6,'ErrorControl',260) -ServiceInstall.add_field(7,'LoadOrderGroup',7679) -ServiceInstall.add_field(8,'Dependencies',7679) -ServiceInstall.add_field(9,'StartName',7679) -ServiceInstall.add_field(10,'Password',7679) -ServiceInstall.add_field(11,'Arguments',7679) -ServiceInstall.add_field(12,'Component_',3400) -ServiceInstall.add_field(13,'Description',8191) - -Shortcut = Table('Shortcut') -Shortcut.add_field(1,'Shortcut',11592) -Shortcut.add_field(2,'Directory_',3400) -Shortcut.add_field(3,'Name',3968) -Shortcut.add_field(4,'Component_',3400) -Shortcut.add_field(5,'Target',3400) -Shortcut.add_field(6,'Arguments',7679) -Shortcut.add_field(7,'Description',8191) -Shortcut.add_field(8,'Hotkey',5378) -Shortcut.add_field(9,'Icon_',7496) -Shortcut.add_field(10,'IconIndex',5378) -Shortcut.add_field(11,'ShowCmd',5378) -Shortcut.add_field(12,'WkDir',7496) - -Signature = Table('Signature') -Signature.add_field(1,'Signature',11592) -Signature.add_field(2,'FileName',3583) -Signature.add_field(3,'MinVersion',7444) -Signature.add_field(4,'MaxVersion',7444) -Signature.add_field(5,'MinSize',4356) -Signature.add_field(6,'MaxSize',4356) -Signature.add_field(7,'MinDate',4356) -Signature.add_field(8,'MaxDate',4356) -Signature.add_field(9,'Languages',7679) - -TextStyle = Table('TextStyle') -TextStyle.add_field(1,'TextStyle',11592) -TextStyle.add_field(2,'FaceName',3360) -TextStyle.add_field(3,'Size',1282) -TextStyle.add_field(4,'Color',4356) -TextStyle.add_field(5,'StyleBits',5378) - -TypeLib = Table('TypeLib') -TypeLib.add_field(1,'LibID',11558) -TypeLib.add_field(2,'Language',9474) -TypeLib.add_field(3,'Component_',11592) -TypeLib.add_field(4,'Version',4356) -TypeLib.add_field(5,'Description',8064) -TypeLib.add_field(6,'Directory_',7496) -TypeLib.add_field(7,'Feature_',3366) -TypeLib.add_field(8,'Cost',4356) - -UIText = Table('UIText') -UIText.add_field(1,'Key',11592) -UIText.add_field(2,'Text',8191) - -Upgrade = Table('Upgrade') -Upgrade.add_field(1,'UpgradeCode',11558) -Upgrade.add_field(2,'VersionMin',15636) -Upgrade.add_field(3,'VersionMax',15636) -Upgrade.add_field(4,'Language',15871) -Upgrade.add_field(5,'Attributes',8452) -Upgrade.add_field(6,'Remove',7679) -Upgrade.add_field(7,'ActionProperty',3400) - -Verb = Table('Verb') -Verb.add_field(1,'Extension_',11775) -Verb.add_field(2,'Verb',11552) -Verb.add_field(3,'Sequence',5378) -Verb.add_field(4,'Command',8191) -Verb.add_field(5,'Argument',8191) - -tables=[_Validation, ActionText, AdminExecuteSequence, Condition, AdminUISequence, AdvtExecuteSequence, AdvtUISequence, AppId, AppSearch, Property, BBControl, Billboard, Feature, Binary, BindImage, File, CCPSearch, CheckBox, Class, Component, Icon, ProgId, ComboBox, CompLocator, Complus, Directory, Control, Dialog, ControlCondition, ControlEvent, CreateFolder, CustomAction, DrLocator, DuplicateFile, Environment, Error, EventMapping, Extension, MIME, FeatureComponents, FileSFPCatalog, SFPCatalog, Font, IniFile, IniLocator, InstallExecuteSequence, InstallUISequence, IsolatedComponent, LaunchCondition, ListBox, ListView, LockPermissions, Media, MoveFile, MsiAssembly, MsiAssemblyName, MsiDigitalCertificate, MsiDigitalSignature, MsiFileHash, MsiPatchHeaders, ODBCAttribute, ODBCDriver, ODBCDataSource, ODBCSourceAttribute, ODBCTranslator, Patch, PatchPackage, PublishComponent, RadioButton, Registry, RegLocator, RemoveFile, RemoveIniFile, RemoveRegistry, ReserveCost, SelfReg, ServiceControl, ServiceInstall, Shortcut, Signature, TextStyle, TypeLib, UIText, Upgrade, Verb] - -_Validation_records = [ -('_Validation','Table','N',None, None, None, None, 'Identifier',None, 'Name of table',), -('_Validation','Column','N',None, None, None, None, 'Identifier',None, 'Name of column',), -('_Validation','Description','Y',None, None, None, None, 'Text',None, 'Description of column',), -('_Validation','Set','Y',None, None, None, None, 'Text',None, 'Set of values that are permitted',), -('_Validation','Category','Y',None, None, None, None, None, 'Text;Formatted;Template;Condition;Guid;Path;Version;Language;Identifier;Binary;UpperCase;LowerCase;Filename;Paths;AnyPath;WildCardFilename;RegPath;KeyFormatted;CustomSource;Property;Cabinet;Shortcut;URL','String category',), -('_Validation','KeyColumn','Y',1,32,None, None, None, None, 'Column to which foreign key connects',), -('_Validation','KeyTable','Y',None, None, None, None, 'Identifier',None, 'For foreign key, Name of table to which data must link',), -('_Validation','MaxValue','Y',-2147483647,2147483647,None, None, None, None, 'Maximum value allowed',), -('_Validation','MinValue','Y',-2147483647,2147483647,None, None, None, None, 'Minimum value allowed',), -('_Validation','Nullable','N',None, None, None, None, None, 'Y;N;@','Whether the column is nullable',), -('ActionText','Description','Y',None, None, None, None, 'Text',None, 'Localized description displayed in progress dialog and log when action is executing.',), -('ActionText','Action','N',None, None, None, None, 'Identifier',None, 'Name of action to be described.',), -('ActionText','Template','Y',None, None, None, None, 'Template',None, 'Optional localized format template used to format action data records for display during action execution.',), -('AdminExecuteSequence','Action','N',None, None, None, None, 'Identifier',None, 'Name of action to invoke, either in the engine or the handler DLL.',), -('AdminExecuteSequence','Condition','Y',None, None, None, None, 'Condition',None, 'Optional expression which skips the action if evaluates to expFalse.If the expression syntax is invalid, the engine will terminate, returning iesBadActionData.',), -('AdminExecuteSequence','Sequence','Y',-4,32767,None, None, None, None, 'Number that determines the sort order in which the actions are to be executed. Leave blank to suppress action.',), -('Condition','Condition','Y',None, None, None, None, 'Condition',None, 'Expression evaluated to determine if Level in the Feature table is to change.',), -('Condition','Feature_','N',None, None, 'Feature',1,'Identifier',None, 'Reference to a Feature entry in Feature table.',), -('Condition','Level','N',0,32767,None, None, None, None, 'New selection Level to set in Feature table if Condition evaluates to TRUE.',), -('AdminUISequence','Action','N',None, None, None, None, 'Identifier',None, 'Name of action to invoke, either in the engine or the handler DLL.',), -('AdminUISequence','Condition','Y',None, None, None, None, 'Condition',None, 'Optional expression which skips the action if evaluates to expFalse.If the expression syntax is invalid, the engine will terminate, returning iesBadActionData.',), -('AdminUISequence','Sequence','Y',-4,32767,None, None, None, None, 'Number that determines the sort order in which the actions are to be executed. Leave blank to suppress action.',), -('AdvtExecuteSequence','Action','N',None, None, None, None, 'Identifier',None, 'Name of action to invoke, either in the engine or the handler DLL.',), -('AdvtExecuteSequence','Condition','Y',None, None, None, None, 'Condition',None, 'Optional expression which skips the action if evaluates to expFalse.If the expression syntax is invalid, the engine will terminate, returning iesBadActionData.',), -('AdvtExecuteSequence','Sequence','Y',-4,32767,None, None, None, None, 'Number that determines the sort order in which the actions are to be executed. Leave blank to suppress action.',), -('AdvtUISequence','Action','N',None, None, None, None, 'Identifier',None, 'Name of action to invoke, either in the engine or the handler DLL.',), -('AdvtUISequence','Condition','Y',None, None, None, None, 'Condition',None, 'Optional expression which skips the action if evaluates to expFalse.If the expression syntax is invalid, the engine will terminate, returning iesBadActionData.',), -('AdvtUISequence','Sequence','Y',-4,32767,None, None, None, None, 'Number that determines the sort order in which the actions are to be executed. Leave blank to suppress action.',), -('AppId','AppId','N',None, None, None, None, 'Guid',None, None, ), -('AppId','ActivateAtStorage','Y',0,1,None, None, None, None, None, ), -('AppId','DllSurrogate','Y',None, None, None, None, 'Text',None, None, ), -('AppId','LocalService','Y',None, None, None, None, 'Text',None, None, ), -('AppId','RemoteServerName','Y',None, None, None, None, 'Formatted',None, None, ), -('AppId','RunAsInteractiveUser','Y',0,1,None, None, None, None, None, ), -('AppId','ServiceParameters','Y',None, None, None, None, 'Text',None, None, ), -('AppSearch','Property','N',None, None, None, None, 'Identifier',None, 'The property associated with a Signature',), -('AppSearch','Signature_','N',None, None, 'Signature;RegLocator;IniLocator;DrLocator;CompLocator',1,'Identifier',None, 'The Signature_ represents a unique file signature and is also the foreign key in the Signature, RegLocator, IniLocator, CompLocator and the DrLocator tables.',), -('Property','Property','N',None, None, None, None, 'Identifier',None, 'Name of property, uppercase if settable by launcher or loader.',), -('Property','Value','N',None, None, None, None, 'Text',None, 'String value for property. Never null or empty.',), -('BBControl','Type','N',None, None, None, None, 'Identifier',None, 'The type of the control.',), -('BBControl','Y','N',0,32767,None, None, None, None, 'Vertical coordinate of the upper left corner of the bounding rectangle of the control.',), -('BBControl','Text','Y',None, None, None, None, 'Text',None, 'A string used to set the initial text contained within a control (if appropriate).',), -('BBControl','BBControl','N',None, None, None, None, 'Identifier',None, 'Name of the control. This name must be unique within a billboard, but can repeat on different billboard.',), -('BBControl','Attributes','Y',0,2147483647,None, None, None, None, 'A 32-bit word that specifies the attribute flags to be applied to this control.',), -('BBControl','Billboard_','N',None, None, 'Billboard',1,'Identifier',None, 'External key to the Billboard table, name of the billboard.',), -('BBControl','Height','N',0,32767,None, None, None, None, 'Height of the bounding rectangle of the control.',), -('BBControl','Width','N',0,32767,None, None, None, None, 'Width of the bounding rectangle of the control.',), -('BBControl','X','N',0,32767,None, None, None, None, 'Horizontal coordinate of the upper left corner of the bounding rectangle of the control.',), -('Billboard','Action','Y',None, None, None, None, 'Identifier',None, 'The name of an action. The billboard is displayed during the progress messages received from this action.',), -('Billboard','Billboard','N',None, None, None, None, 'Identifier',None, 'Name of the billboard.',), -('Billboard','Feature_','N',None, None, 'Feature',1,'Identifier',None, 'An external key to the Feature Table. The billboard is shown only if this feature is being installed.',), -('Billboard','Ordering','Y',0,32767,None, None, None, None, 'A positive integer. If there is more than one billboard corresponding to an action they will be shown in the order defined by this column.',), -('Feature','Description','Y',None, None, None, None, 'Text',None, 'Longer descriptive text describing a visible feature item.',), -('Feature','Attributes','N',None, None, None, None, None, '0;1;2;4;5;6;8;9;10;16;17;18;20;21;22;24;25;26;32;33;34;36;37;38;48;49;50;52;53;54','Feature attributes',), -('Feature','Feature','N',None, None, None, None, 'Identifier',None, 'Primary key used to identify a particular feature record.',), -('Feature','Directory_','Y',None, None, 'Directory',1,'UpperCase',None, 'The name of the Directory that can be configured by the UI. A non-null value will enable the browse button.',), -('Feature','Level','N',0,32767,None, None, None, None, 'The install level at which record will be initially selected. An install level of 0 will disable an item and prevent its display.',), -('Feature','Title','Y',None, None, None, None, 'Text',None, 'Short text identifying a visible feature item.',), -('Feature','Display','Y',0,32767,None, None, None, None, 'Numeric sort order, used to force a specific display ordering.',), -('Feature','Feature_Parent','Y',None, None, 'Feature',1,'Identifier',None, 'Optional key of a parent record in the same table. If the parent is not selected, then the record will not be installed. Null indicates a root item.',), -('Binary','Name','N',None, None, None, None, 'Identifier',None, 'Unique key identifying the binary data.',), -('Binary','Data','N',None, None, None, None, 'Binary',None, 'The unformatted binary data.',), -('BindImage','File_','N',None, None, 'File',1,'Identifier',None, 'The index into the File table. This must be an executable file.',), -('BindImage','Path','Y',None, None, None, None, 'Paths',None, 'A list of ; delimited paths that represent the paths to be searched for the import DLLS. The list is usually a list of properties each enclosed within square brackets [] .',), -('File','Sequence','N',1,32767,None, None, None, None, 'Sequence with respect to the media images; order must track cabinet order.',), -('File','Attributes','Y',0,32767,None, None, None, None, 'Integer containing bit flags representing file attributes (with the decimal value of each bit position in parentheses)',), -('File','File','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token, must match identifier in cabinet. For uncompressed files, this field is ignored.',), -('File','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key referencing Component that controls the file.',), -('File','FileName','N',None, None, None, None, 'Filename',None, 'File name used for installation, may be localized. This may contain a "short name|long name" pair.',), -('File','FileSize','N',0,2147483647,None, None, None, None, 'Size of file in bytes (integer).',), -('File','Language','Y',None, None, None, None, 'Language',None, 'List of decimal language Ids, comma-separated if more than one.',), -('File','Version','Y',None, None, 'File',1,'Version',None, 'Version string for versioned files; Blank for unversioned files.',), -('CCPSearch','Signature_','N',None, None, 'Signature;RegLocator;IniLocator;DrLocator;CompLocator',1,'Identifier',None, 'The Signature_ represents a unique file signature and is also the foreign key in the Signature, RegLocator, IniLocator, CompLocator and the DrLocator tables.',), -('CheckBox','Property','N',None, None, None, None, 'Identifier',None, 'A named property to be tied to the item.',), -('CheckBox','Value','Y',None, None, None, None, 'Formatted',None, 'The value string associated with the item.',), -('Class','Description','Y',None, None, None, None, 'Text',None, 'Localized description for the Class.',), -('Class','Attributes','Y',None, 32767,None, None, None, None, 'Class registration attributes.',), -('Class','Feature_','N',None, None, 'Feature',1,'Identifier',None, 'Required foreign key into the Feature Table, specifying the feature to validate or install in order for the CLSID factory to be operational.',), -('Class','AppId_','Y',None, None, 'AppId',1,'Guid',None, 'Optional AppID containing DCOM information for associated application (string GUID).',), -('Class','Argument','Y',None, None, None, None, 'Formatted',None, 'optional argument for LocalServers.',), -('Class','CLSID','N',None, None, None, None, 'Guid',None, 'The CLSID of an OLE factory.',), -('Class','Component_','N',None, None, 'Component',1,'Identifier',None, 'Required foreign key into the Component Table, specifying the component for which to return a path when called through LocateComponent.',), -('Class','Context','N',None, None, None, None, 'Identifier',None, 'The numeric server context for this server. CLSCTX_xxxx',), -('Class','DefInprocHandler','Y',None, None, None, None, 'Filename','1;2;3','Optional default inproc handler. Only optionally provided if Context=CLSCTX_LOCAL_SERVER. Typically "ole32.dll" or "mapi32.dll"',), -('Class','FileTypeMask','Y',None, None, None, None, 'Text',None, 'Optional string containing information for the HKCRthis CLSID key. If multiple patterns exist, they must be delimited by a semicolon, and numeric subkeys will be generated: 0,1,2...',), -('Class','Icon_','Y',None, None, 'Icon',1,'Identifier',None, 'Optional foreign key into the Icon Table, specifying the icon file associated with this CLSID. Will be written under the DefaultIcon key.',), -('Class','IconIndex','Y',-32767,32767,None, None, None, None, 'Optional icon index.',), -('Class','ProgId_Default','Y',None, None, 'ProgId',1,'Text',None, 'Optional ProgId associated with this CLSID.',), -('Component','Condition','Y',None, None, None, None, 'Condition',None, "A conditional statement that will disable this component if the specified condition evaluates to the 'True' state. If a component is disabled, it will not be installed, regardless of the 'Action' state associated with the component.",), -('Component','Attributes','N',None, None, None, None, None, None, 'Remote execution option, one of irsEnum',), -('Component','Component','N',None, None, None, None, 'Identifier',None, 'Primary key used to identify a particular component record.',), -('Component','ComponentId','Y',None, None, None, None, 'Guid',None, 'A string GUID unique to this component, version, and language.',), -('Component','Directory_','N',None, None, 'Directory',1,'Identifier',None, 'Required key of a Directory table record. This is actually a property name whose value contains the actual path, set either by the AppSearch action or with the default setting obtained from the Directory table.',), -('Component','KeyPath','Y',None, None, 'File;Registry;ODBCDataSource',1,'Identifier',None, 'Either the primary key into the File table, Registry table, or ODBCDataSource table. This extract path is stored when the component is installed, and is used to detect the presence of the component and to return the path to it.',), -('Icon','Name','N',None, None, None, None, 'Identifier',None, 'Primary key. Name of the icon file.',), -('Icon','Data','N',None, None, None, None, 'Binary',None, 'Binary stream. The binary icon data in PE (.DLL or .EXE) or icon (.ICO) format.',), -('ProgId','Description','Y',None, None, None, None, 'Text',None, 'Localized description for the Program identifier.',), -('ProgId','Icon_','Y',None, None, 'Icon',1,'Identifier',None, 'Optional foreign key into the Icon Table, specifying the icon file associated with this ProgId. Will be written under the DefaultIcon key.',), -('ProgId','IconIndex','Y',-32767,32767,None, None, None, None, 'Optional icon index.',), -('ProgId','ProgId','N',None, None, None, None, 'Text',None, 'The Program Identifier. Primary key.',), -('ProgId','Class_','Y',None, None, 'Class',1,'Guid',None, 'The CLSID of an OLE factory corresponding to the ProgId.',), -('ProgId','ProgId_Parent','Y',None, None, 'ProgId',1,'Text',None, 'The Parent Program Identifier. If specified, the ProgId column becomes a version independent prog id.',), -('ComboBox','Text','Y',None, None, None, None, 'Formatted',None, 'The visible text to be assigned to the item. Optional. If this entry or the entire column is missing, the text is the same as the value.',), -('ComboBox','Property','N',None, None, None, None, 'Identifier',None, 'A named property to be tied to this item. All the items tied to the same property become part of the same combobox.',), -('ComboBox','Value','N',None, None, None, None, 'Formatted',None, 'The value string associated with this item. Selecting the line will set the associated property to this value.',), -('ComboBox','Order','N',1,32767,None, None, None, None, 'A positive integer used to determine the ordering of the items within one list.\tThe integers do not have to be consecutive.',), -('CompLocator','Type','Y',0,1,None, None, None, None, 'A boolean value that determines if the registry value is a filename or a directory location.',), -('CompLocator','Signature_','N',None, None, None, None, 'Identifier',None, 'The table key. The Signature_ represents a unique file signature and is also the foreign key in the Signature table.',), -('CompLocator','ComponentId','N',None, None, None, None, 'Guid',None, 'A string GUID unique to this component, version, and language.',), -('Complus','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key referencing Component that controls the ComPlus component.',), -('Complus','ExpType','Y',0,32767,None, None, None, None, 'ComPlus component attributes.',), -('Directory','Directory','N',None, None, None, None, 'Identifier',None, 'Unique identifier for directory entry, primary key. If a property by this name is defined, it contains the full path to the directory.',), -('Directory','DefaultDir','N',None, None, None, None, 'DefaultDir',None, "The default sub-path under parent's path.",), -('Directory','Directory_Parent','Y',None, None, 'Directory',1,'Identifier',None, 'Reference to the entry in this table specifying the default parent directory. A record parented to itself or with a Null parent represents a root of the install tree.',), -('Control','Type','N',None, None, None, None, 'Identifier',None, 'The type of the control.',), -('Control','Y','N',0,32767,None, None, None, None, 'Vertical coordinate of the upper left corner of the bounding rectangle of the control.',), -('Control','Text','Y',None, None, None, None, 'Formatted',None, 'A string used to set the initial text contained within a control (if appropriate).',), -('Control','Property','Y',None, None, None, None, 'Identifier',None, 'The name of a defined property to be linked to this control. ',), -('Control','Attributes','Y',0,2147483647,None, None, None, None, 'A 32-bit word that specifies the attribute flags to be applied to this control.',), -('Control','Height','N',0,32767,None, None, None, None, 'Height of the bounding rectangle of the control.',), -('Control','Width','N',0,32767,None, None, None, None, 'Width of the bounding rectangle of the control.',), -('Control','X','N',0,32767,None, None, None, None, 'Horizontal coordinate of the upper left corner of the bounding rectangle of the control.',), -('Control','Control','N',None, None, None, None, 'Identifier',None, 'Name of the control. This name must be unique within a dialog, but can repeat on different dialogs. ',), -('Control','Control_Next','Y',None, None, 'Control',2,'Identifier',None, 'The name of an other control on the same dialog. This link defines the tab order of the controls. The links have to form one or more cycles!',), -('Control','Dialog_','N',None, None, 'Dialog',1,'Identifier',None, 'External key to the Dialog table, name of the dialog.',), -('Control','Help','Y',None, None, None, None, 'Text',None, 'The help strings used with the button. The text is optional. ',), -('Dialog','Attributes','Y',0,2147483647,None, None, None, None, 'A 32-bit word that specifies the attribute flags to be applied to this dialog.',), -('Dialog','Height','N',0,32767,None, None, None, None, 'Height of the bounding rectangle of the dialog.',), -('Dialog','Width','N',0,32767,None, None, None, None, 'Width of the bounding rectangle of the dialog.',), -('Dialog','Dialog','N',None, None, None, None, 'Identifier',None, 'Name of the dialog.',), -('Dialog','Control_Cancel','Y',None, None, 'Control',2,'Identifier',None, 'Defines the cancel control. Hitting escape or clicking on the close icon on the dialog is equivalent to pushing this button.',), -('Dialog','Control_Default','Y',None, None, 'Control',2,'Identifier',None, 'Defines the default control. Hitting return is equivalent to pushing this button.',), -('Dialog','Control_First','N',None, None, 'Control',2,'Identifier',None, 'Defines the control that has the focus when the dialog is created.',), -('Dialog','HCentering','N',0,100,None, None, None, None, 'Horizontal position of the dialog on a 0-100 scale. 0 means left end, 100 means right end of the screen, 50 center.',), -('Dialog','Title','Y',None, None, None, None, 'Formatted',None, "A text string specifying the title to be displayed in the title bar of the dialog's window.",), -('Dialog','VCentering','N',0,100,None, None, None, None, 'Vertical position of the dialog on a 0-100 scale. 0 means top end, 100 means bottom end of the screen, 50 center.',), -('ControlCondition','Action','N',None, None, None, None, None, 'Default;Disable;Enable;Hide;Show','The desired action to be taken on the specified control.',), -('ControlCondition','Condition','N',None, None, None, None, 'Condition',None, 'A standard conditional statement that specifies under which conditions the action should be triggered.',), -('ControlCondition','Dialog_','N',None, None, 'Dialog',1,'Identifier',None, 'A foreign key to the Dialog table, name of the dialog.',), -('ControlCondition','Control_','N',None, None, 'Control',2,'Identifier',None, 'A foreign key to the Control table, name of the control.',), -('ControlEvent','Condition','Y',None, None, None, None, 'Condition',None, 'A standard conditional statement that specifies under which conditions an event should be triggered.',), -('ControlEvent','Ordering','Y',0,2147483647,None, None, None, None, 'An integer used to order several events tied to the same control. Can be left blank.',), -('ControlEvent','Argument','N',None, None, None, None, 'Formatted',None, 'A value to be used as a modifier when triggering a particular event.',), -('ControlEvent','Dialog_','N',None, None, 'Dialog',1,'Identifier',None, 'A foreign key to the Dialog table, name of the dialog.',), -('ControlEvent','Control_','N',None, None, 'Control',2,'Identifier',None, 'A foreign key to the Control table, name of the control',), -('ControlEvent','Event','N',None, None, None, None, 'Formatted',None, 'An identifier that specifies the type of the event that should take place when the user interacts with control specified by the first two entries.',), -('CreateFolder','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into the Component table.',), -('CreateFolder','Directory_','N',None, None, 'Directory',1,'Identifier',None, 'Primary key, could be foreign key into the Directory table.',), -('CustomAction','Type','N',1,16383,None, None, None, None, 'The numeric custom action type, consisting of source location, code type, entry, option flags.',), -('CustomAction','Action','N',None, None, None, None, 'Identifier',None, 'Primary key, name of action, normally appears in sequence table unless private use.',), -('CustomAction','Source','Y',None, None, None, None, 'CustomSource',None, 'The table reference of the source of the code.',), -('CustomAction','Target','Y',None, None, None, None, 'Formatted',None, 'Execution parameter, depends on the type of custom action',), -('DrLocator','Signature_','N',None, None, None, None, 'Identifier',None, 'The Signature_ represents a unique file signature and is also the foreign key in the Signature table.',), -('DrLocator','Path','Y',None, None, None, None, 'AnyPath',None, 'The path on the user system. This is either a subpath below the value of the Parent or a full path. The path may contain properties enclosed within [ ] that will be expanded.',), -('DrLocator','Depth','Y',0,32767,None, None, None, None, 'The depth below the path to which the Signature_ is recursively searched. If absent, the depth is assumed to be 0.',), -('DrLocator','Parent','Y',None, None, None, None, 'Identifier',None, 'The parent file signature. It is also a foreign key in the Signature table. If null and the Path column does not expand to a full path, then all the fixed drives of the user system are searched using the Path.',), -('DuplicateFile','File_','N',None, None, 'File',1,'Identifier',None, 'Foreign key referencing the source file to be duplicated.',), -('DuplicateFile','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key referencing Component that controls the duplicate file.',), -('DuplicateFile','DestFolder','Y',None, None, None, None, 'Identifier',None, 'Name of a property whose value is assumed to resolve to the full pathname to a destination folder.',), -('DuplicateFile','DestName','Y',None, None, None, None, 'Filename',None, 'Filename to be given to the duplicate file.',), -('DuplicateFile','FileKey','N',None, None, None, None, 'Identifier',None, 'Primary key used to identify a particular file entry',), -('Environment','Name','N',None, None, None, None, 'Text',None, 'The name of the environmental value.',), -('Environment','Value','Y',None, None, None, None, 'Formatted',None, 'The value to set in the environmental settings.',), -('Environment','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into the Component table referencing component that controls the installing of the environmental value.',), -('Environment','Environment','N',None, None, None, None, 'Identifier',None, 'Unique identifier for the environmental variable setting',), -('Error','Error','N',0,32767,None, None, None, None, 'Integer error number, obtained from header file IError(...) macros.',), -('Error','Message','Y',None, None, None, None, 'Template',None, 'Error formatting template, obtained from user ed. or localizers.',), -('EventMapping','Dialog_','N',None, None, 'Dialog',1,'Identifier',None, 'A foreign key to the Dialog table, name of the Dialog.',), -('EventMapping','Control_','N',None, None, 'Control',2,'Identifier',None, 'A foreign key to the Control table, name of the control.',), -('EventMapping','Event','N',None, None, None, None, 'Identifier',None, 'An identifier that specifies the type of the event that the control subscribes to.',), -('EventMapping','Attribute','N',None, None, None, None, 'Identifier',None, 'The name of the control attribute, that is set when this event is received.',), -('Extension','Feature_','N',None, None, 'Feature',1,'Identifier',None, 'Required foreign key into the Feature Table, specifying the feature to validate or install in order for the CLSID factory to be operational.',), -('Extension','Component_','N',None, None, 'Component',1,'Identifier',None, 'Required foreign key into the Component Table, specifying the component for which to return a path when called through LocateComponent.',), -('Extension','Extension','N',None, None, None, None, 'Text',None, 'The extension associated with the table row.',), -('Extension','MIME_','Y',None, None, 'MIME',1,'Text',None, 'Optional Context identifier, typically "type/format" associated with the extension',), -('Extension','ProgId_','Y',None, None, 'ProgId',1,'Text',None, 'Optional ProgId associated with this extension.',), -('MIME','CLSID','Y',None, None, None, None, 'Guid',None, 'Optional associated CLSID.',), -('MIME','ContentType','N',None, None, None, None, 'Text',None, 'Primary key. Context identifier, typically "type/format".',), -('MIME','Extension_','N',None, None, 'Extension',1,'Text',None, 'Optional associated extension (without dot)',), -('FeatureComponents','Feature_','N',None, None, 'Feature',1,'Identifier',None, 'Foreign key into Feature table.',), -('FeatureComponents','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into Component table.',), -('FileSFPCatalog','File_','N',None, None, 'File',1,'Identifier',None, 'File associated with the catalog',), -('FileSFPCatalog','SFPCatalog_','N',None, None, 'SFPCatalog',1,'Filename',None, 'Catalog associated with the file',), -('SFPCatalog','SFPCatalog','N',None, None, None, None, 'Filename',None, 'File name for the catalog.',), -('SFPCatalog','Catalog','N',None, None, None, None, 'Binary',None, 'SFP Catalog',), -('SFPCatalog','Dependency','Y',None, None, None, None, 'Formatted',None, 'Parent catalog - only used by SFP',), -('Font','File_','N',None, None, 'File',1,'Identifier',None, 'Primary key, foreign key into File table referencing font file.',), -('Font','FontTitle','Y',None, None, None, None, 'Text',None, 'Font name.',), -('IniFile','Action','N',None, None, None, None, None, '0;1;3','The type of modification to be made, one of iifEnum',), -('IniFile','Value','N',None, None, None, None, 'Formatted',None, 'The value to be written.',), -('IniFile','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into the Component table referencing component that controls the installing of the .INI value.',), -('IniFile','FileName','N',None, None, None, None, 'Filename',None, 'The .INI file name in which to write the information',), -('IniFile','IniFile','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token.',), -('IniFile','DirProperty','Y',None, None, None, None, 'Identifier',None, 'Foreign key into the Directory table denoting the directory where the .INI file is.',), -('IniFile','Key','N',None, None, None, None, 'Formatted',None, 'The .INI file key below Section.',), -('IniFile','Section','N',None, None, None, None, 'Formatted',None, 'The .INI file Section.',), -('IniLocator','Type','Y',0,2,None, None, None, None, 'An integer value that determines if the .INI value read is a filename or a directory location or to be used as is w/o interpretation.',), -('IniLocator','Signature_','N',None, None, None, None, 'Identifier',None, 'The table key. The Signature_ represents a unique file signature and is also the foreign key in the Signature table.',), -('IniLocator','FileName','N',None, None, None, None, 'Filename',None, 'The .INI file name.',), -('IniLocator','Key','N',None, None, None, None, 'Text',None, 'Key value (followed by an equals sign in INI file).',), -('IniLocator','Section','N',None, None, None, None, 'Text',None, 'Section name within in file (within square brackets in INI file).',), -('IniLocator','Field','Y',0,32767,None, None, None, None, 'The field in the .INI line. If Field is null or 0 the entire line is read.',), -('InstallExecuteSequence','Action','N',None, None, None, None, 'Identifier',None, 'Name of action to invoke, either in the engine or the handler DLL.',), -('InstallExecuteSequence','Condition','Y',None, None, None, None, 'Condition',None, 'Optional expression which skips the action if evaluates to expFalse.If the expression syntax is invalid, the engine will terminate, returning iesBadActionData.',), -('InstallExecuteSequence','Sequence','Y',-4,32767,None, None, None, None, 'Number that determines the sort order in which the actions are to be executed. Leave blank to suppress action.',), -('InstallUISequence','Action','N',None, None, None, None, 'Identifier',None, 'Name of action to invoke, either in the engine or the handler DLL.',), -('InstallUISequence','Condition','Y',None, None, None, None, 'Condition',None, 'Optional expression which skips the action if evaluates to expFalse.If the expression syntax is invalid, the engine will terminate, returning iesBadActionData.',), -('InstallUISequence','Sequence','Y',-4,32767,None, None, None, None, 'Number that determines the sort order in which the actions are to be executed. Leave blank to suppress action.',), -('IsolatedComponent','Component_Application','N',None, None, 'Component',1,'Identifier',None, 'Key to Component table item for application',), -('IsolatedComponent','Component_Shared','N',None, None, 'Component',1,'Identifier',None, 'Key to Component table item to be isolated',), -('LaunchCondition','Description','N',None, None, None, None, 'Formatted',None, 'Localizable text to display when condition fails and install must abort.',), -('LaunchCondition','Condition','N',None, None, None, None, 'Condition',None, 'Expression which must evaluate to TRUE in order for install to commence.',), -('ListBox','Text','Y',None, None, None, None, 'Text',None, 'The visible text to be assigned to the item. Optional. If this entry or the entire column is missing, the text is the same as the value.',), -('ListBox','Property','N',None, None, None, None, 'Identifier',None, 'A named property to be tied to this item. All the items tied to the same property become part of the same listbox.',), -('ListBox','Value','N',None, None, None, None, 'Formatted',None, 'The value string associated with this item. Selecting the line will set the associated property to this value.',), -('ListBox','Order','N',1,32767,None, None, None, None, 'A positive integer used to determine the ordering of the items within one list..The integers do not have to be consecutive.',), -('ListView','Text','Y',None, None, None, None, 'Text',None, 'The visible text to be assigned to the item. Optional. If this entry or the entire column is missing, the text is the same as the value.',), -('ListView','Property','N',None, None, None, None, 'Identifier',None, 'A named property to be tied to this item. All the items tied to the same property become part of the same listview.',), -('ListView','Value','N',None, None, None, None, 'Identifier',None, 'The value string associated with this item. Selecting the line will set the associated property to this value.',), -('ListView','Order','N',1,32767,None, None, None, None, 'A positive integer used to determine the ordering of the items within one list..The integers do not have to be consecutive.',), -('ListView','Binary_','Y',None, None, 'Binary',1,'Identifier',None, 'The name of the icon to be displayed with the icon. The binary information is looked up from the Binary Table.',), -('LockPermissions','Table','N',None, None, None, None, 'Identifier','Directory;File;Registry','Reference to another table name',), -('LockPermissions','Domain','Y',None, None, None, None, 'Formatted',None, 'Domain name for user whose permissions are being set. (usually a property)',), -('LockPermissions','LockObject','N',None, None, None, None, 'Identifier',None, 'Foreign key into Registry or File table',), -('LockPermissions','Permission','Y',-2147483647,2147483647,None, None, None, None, 'Permission Access mask. Full Control = 268435456 (GENERIC_ALL = 0x10000000)',), -('LockPermissions','User','N',None, None, None, None, 'Formatted',None, 'User for permissions to be set. (usually a property)',), -('Media','Source','Y',None, None, None, None, 'Property',None, 'The property defining the location of the cabinet file.',), -('Media','Cabinet','Y',None, None, None, None, 'Cabinet',None, 'If some or all of the files stored on the media are compressed in a cabinet, the name of that cabinet.',), -('Media','DiskId','N',1,32767,None, None, None, None, 'Primary key, integer to determine sort order for table.',), -('Media','DiskPrompt','Y',None, None, None, None, 'Text',None, 'Disk name: the visible text actually printed on the disk. This will be used to prompt the user when this disk needs to be inserted.',), -('Media','LastSequence','N',0,32767,None, None, None, None, 'File sequence number for the last file for this media.',), -('Media','VolumeLabel','Y',None, None, None, None, 'Text',None, 'The label attributed to the volume.',), -('ModuleComponents','Component','N',None, None, 'Component',1,'Identifier',None, 'Component contained in the module.',), -('ModuleComponents','Language','N',None, None, 'ModuleSignature',2,None, None, 'Default language ID for module (may be changed by transform).',), -('ModuleComponents','ModuleID','N',None, None, 'ModuleSignature',1,'Identifier',None, 'Module containing the component.',), -('ModuleSignature','Language','N',None, None, None, None, None, None, 'Default decimal language of module.',), -('ModuleSignature','Version','N',None, None, None, None, 'Version',None, 'Version of the module.',), -('ModuleSignature','ModuleID','N',None, None, None, None, 'Identifier',None, 'Module identifier (String.GUID).',), -('ModuleDependency','ModuleID','N',None, None, 'ModuleSignature',1,'Identifier',None, 'Module requiring the dependency.',), -('ModuleDependency','ModuleLanguage','N',None, None, 'ModuleSignature',2,None, None, 'Language of module requiring the dependency.',), -('ModuleDependency','RequiredID','N',None, None, None, None, None, None, 'String.GUID of required module.',), -('ModuleDependency','RequiredLanguage','N',None, None, None, None, None, None, 'LanguageID of the required module.',), -('ModuleDependency','RequiredVersion','Y',None, None, None, None, 'Version',None, 'Version of the required version.',), -('ModuleExclusion','ModuleID','N',None, None, 'ModuleSignature',1,'Identifier',None, 'String.GUID of module with exclusion requirement.',), -('ModuleExclusion','ModuleLanguage','N',None, None, 'ModuleSignature',2,None, None, 'LanguageID of module with exclusion requirement.',), -('ModuleExclusion','ExcludedID','N',None, None, None, None, None, None, 'String.GUID of excluded module.',), -('ModuleExclusion','ExcludedLanguage','N',None, None, None, None, None, None, 'Language of excluded module.',), -('ModuleExclusion','ExcludedMaxVersion','Y',None, None, None, None, 'Version',None, 'Maximum version of excluded module.',), -('ModuleExclusion','ExcludedMinVersion','Y',None, None, None, None, 'Version',None, 'Minimum version of excluded module.',), -('MoveFile','Component_','N',None, None, 'Component',1,'Identifier',None, 'If this component is not "selected" for installation or removal, no action will be taken on the associated MoveFile entry',), -('MoveFile','DestFolder','N',None, None, None, None, 'Identifier',None, 'Name of a property whose value is assumed to resolve to the full path to the destination directory',), -('MoveFile','DestName','Y',None, None, None, None, 'Filename',None, 'Name to be given to the original file after it is moved or copied. If blank, the destination file will be given the same name as the source file',), -('MoveFile','FileKey','N',None, None, None, None, 'Identifier',None, 'Primary key that uniquely identifies a particular MoveFile record',), -('MoveFile','Options','N',0,1,None, None, None, None, 'Integer value specifying the MoveFile operating mode, one of imfoEnum',), -('MoveFile','SourceFolder','Y',None, None, None, None, 'Identifier',None, 'Name of a property whose value is assumed to resolve to the full path to the source directory',), -('MoveFile','SourceName','Y',None, None, None, None, 'Text',None, "Name of the source file(s) to be moved or copied. Can contain the '*' or '?' wildcards.",), -('MsiAssembly','Attributes','Y',None, None, None, None, None, None, 'Assembly attributes',), -('MsiAssembly','Feature_','N',None, None, 'Feature',1,'Identifier',None, 'Foreign key into Feature table.',), -('MsiAssembly','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into Component table.',), -('MsiAssembly','File_Application','Y',None, None, 'File',1,'Identifier',None, 'Foreign key into File table, denoting the application context for private assemblies. Null for global assemblies.',), -('MsiAssembly','File_Manifest','Y',None, None, 'File',1,'Identifier',None, 'Foreign key into the File table denoting the manifest file for the assembly.',), -('MsiAssemblyName','Name','N',None, None, None, None, 'Text',None, 'The name part of the name-value pairs for the assembly name.',), -('MsiAssemblyName','Value','N',None, None, None, None, 'Text',None, 'The value part of the name-value pairs for the assembly name.',), -('MsiAssemblyName','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into Component table.',), -('MsiDigitalCertificate','CertData','N',None, None, None, None, 'Binary',None, 'A certificate context blob for a signer certificate',), -('MsiDigitalCertificate','DigitalCertificate','N',None, None, None, None, 'Identifier',None, 'A unique identifier for the row',), -('MsiDigitalSignature','Table','N',None, None, None, None, None, 'Media','Reference to another table name (only Media table is supported)',), -('MsiDigitalSignature','DigitalCertificate_','N',None, None, 'MsiDigitalCertificate',1,'Identifier',None, 'Foreign key to MsiDigitalCertificate table identifying the signer certificate',), -('MsiDigitalSignature','Hash','Y',None, None, None, None, 'Binary',None, 'The encoded hash blob from the digital signature',), -('MsiDigitalSignature','SignObject','N',None, None, None, None, 'Text',None, 'Foreign key to Media table',), -('MsiFileHash','File_','N',None, None, 'File',1,'Identifier',None, 'Primary key, foreign key into File table referencing file with this hash',), -('MsiFileHash','Options','N',0,32767,None, None, None, None, 'Various options and attributes for this hash.',), -('MsiFileHash','HashPart1','N',None, None, None, None, None, None, 'Size of file in bytes (integer).',), -('MsiFileHash','HashPart2','N',None, None, None, None, None, None, 'Size of file in bytes (integer).',), -('MsiFileHash','HashPart3','N',None, None, None, None, None, None, 'Size of file in bytes (integer).',), -('MsiFileHash','HashPart4','N',None, None, None, None, None, None, 'Size of file in bytes (integer).',), -('MsiPatchHeaders','StreamRef','N',None, None, None, None, 'Identifier',None, 'Primary key. A unique identifier for the row.',), -('MsiPatchHeaders','Header','N',None, None, None, None, 'Binary',None, 'Binary stream. The patch header, used for patch validation.',), -('ODBCAttribute','Value','Y',None, None, None, None, 'Text',None, 'Value for ODBC driver attribute',), -('ODBCAttribute','Attribute','N',None, None, None, None, 'Text',None, 'Name of ODBC driver attribute',), -('ODBCAttribute','Driver_','N',None, None, 'ODBCDriver',1,'Identifier',None, 'Reference to ODBC driver in ODBCDriver table',), -('ODBCDriver','Description','N',None, None, None, None, 'Text',None, 'Text used as registered name for driver, non-localized',), -('ODBCDriver','File_','N',None, None, 'File',1,'Identifier',None, 'Reference to key driver file',), -('ODBCDriver','Component_','N',None, None, 'Component',1,'Identifier',None, 'Reference to associated component',), -('ODBCDriver','Driver','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized.internal token for driver',), -('ODBCDriver','File_Setup','Y',None, None, 'File',1,'Identifier',None, 'Optional reference to key driver setup DLL',), -('ODBCDataSource','Description','N',None, None, None, None, 'Text',None, 'Text used as registered name for data source',), -('ODBCDataSource','Component_','N',None, None, 'Component',1,'Identifier',None, 'Reference to associated component',), -('ODBCDataSource','DataSource','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized.internal token for data source',), -('ODBCDataSource','DriverDescription','N',None, None, None, None, 'Text',None, 'Reference to driver description, may be existing driver',), -('ODBCDataSource','Registration','N',0,1,None, None, None, None, 'Registration option: 0=machine, 1=user, others t.b.d.',), -('ODBCSourceAttribute','Value','Y',None, None, None, None, 'Text',None, 'Value for ODBC data source attribute',), -('ODBCSourceAttribute','Attribute','N',None, None, None, None, 'Text',None, 'Name of ODBC data source attribute',), -('ODBCSourceAttribute','DataSource_','N',None, None, 'ODBCDataSource',1,'Identifier',None, 'Reference to ODBC data source in ODBCDataSource table',), -('ODBCTranslator','Description','N',None, None, None, None, 'Text',None, 'Text used as registered name for translator',), -('ODBCTranslator','File_','N',None, None, 'File',1,'Identifier',None, 'Reference to key translator file',), -('ODBCTranslator','Component_','N',None, None, 'Component',1,'Identifier',None, 'Reference to associated component',), -('ODBCTranslator','File_Setup','Y',None, None, 'File',1,'Identifier',None, 'Optional reference to key translator setup DLL',), -('ODBCTranslator','Translator','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized.internal token for translator',), -('Patch','Sequence','N',0,32767,None, None, None, None, 'Primary key, sequence with respect to the media images; order must track cabinet order.',), -('Patch','Attributes','N',0,32767,None, None, None, None, 'Integer containing bit flags representing patch attributes',), -('Patch','File_','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token, foreign key to File table, must match identifier in cabinet.',), -('Patch','Header','Y',None, None, None, None, 'Binary',None, 'Binary stream. The patch header, used for patch validation.',), -('Patch','PatchSize','N',0,2147483647,None, None, None, None, 'Size of patch in bytes (integer).',), -('Patch','StreamRef_','Y',None, None, None, None, 'Identifier',None, 'Identifier. Foreign key to the StreamRef column of the MsiPatchHeaders table.',), -('PatchPackage','Media_','N',0,32767,None, None, None, None, 'Foreign key to DiskId column of Media table. Indicates the disk containing the patch package.',), -('PatchPackage','PatchId','N',None, None, None, None, 'Guid',None, 'A unique string GUID representing this patch.',), -('PublishComponent','Feature_','N',None, None, 'Feature',1,'Identifier',None, 'Foreign key into the Feature table.',), -('PublishComponent','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into the Component table.',), -('PublishComponent','ComponentId','N',None, None, None, None, 'Guid',None, 'A string GUID that represents the component id that will be requested by the alien product.',), -('PublishComponent','AppData','Y',None, None, None, None, 'Text',None, 'This is localisable Application specific data that can be associated with a Qualified Component.',), -('PublishComponent','Qualifier','N',None, None, None, None, 'Text',None, 'This is defined only when the ComponentId column is an Qualified Component Id. This is the Qualifier for ProvideComponentIndirect.',), -('RadioButton','Y','N',0,32767,None, None, None, None, 'The vertical coordinate of the upper left corner of the bounding rectangle of the radio button.',), -('RadioButton','Text','Y',None, None, None, None, 'Text',None, 'The visible title to be assigned to the radio button.',), -('RadioButton','Property','N',None, None, None, None, 'Identifier',None, 'A named property to be tied to this radio button. All the buttons tied to the same property become part of the same group.',), -('RadioButton','Height','N',0,32767,None, None, None, None, 'The height of the button.',), -('RadioButton','Width','N',0,32767,None, None, None, None, 'The width of the button.',), -('RadioButton','X','N',0,32767,None, None, None, None, 'The horizontal coordinate of the upper left corner of the bounding rectangle of the radio button.',), -('RadioButton','Value','N',None, None, None, None, 'Formatted',None, 'The value string associated with this button. Selecting the button will set the associated property to this value.',), -('RadioButton','Order','N',1,32767,None, None, None, None, 'A positive integer used to determine the ordering of the items within one list..The integers do not have to be consecutive.',), -('RadioButton','Help','Y',None, None, None, None, 'Text',None, 'The help strings used with the button. The text is optional.',), -('Registry','Name','Y',None, None, None, None, 'Formatted',None, 'The registry value name.',), -('Registry','Value','Y',None, None, None, None, 'Formatted',None, 'The registry value.',), -('Registry','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into the Component table referencing component that controls the installing of the registry value.',), -('Registry','Key','N',None, None, None, None, 'RegPath',None, 'The key for the registry value.',), -('Registry','Registry','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token.',), -('Registry','Root','N',-1,3,None, None, None, None, 'The predefined root key for the registry value, one of rrkEnum.',), -('RegLocator','Name','Y',None, None, None, None, 'Formatted',None, 'The registry value name.',), -('RegLocator','Type','Y',0,18,None, None, None, None, 'An integer value that determines if the registry value is a filename or a directory location or to be used as is w/o interpretation.',), -('RegLocator','Signature_','N',None, None, None, None, 'Identifier',None, 'The table key. The Signature_ represents a unique file signature and is also the foreign key in the Signature table. If the type is 0, the registry values refers a directory, and _Signature is not a foreign key.',), -('RegLocator','Key','N',None, None, None, None, 'RegPath',None, 'The key for the registry value.',), -('RegLocator','Root','N',0,3,None, None, None, None, 'The predefined root key for the registry value, one of rrkEnum.',), -('RemoveFile','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key referencing Component that controls the file to be removed.',), -('RemoveFile','FileKey','N',None, None, None, None, 'Identifier',None, 'Primary key used to identify a particular file entry',), -('RemoveFile','FileName','Y',None, None, None, None, 'WildCardFilename',None, 'Name of the file to be removed.',), -('RemoveFile','DirProperty','N',None, None, None, None, 'Identifier',None, 'Name of a property whose value is assumed to resolve to the full pathname to the folder of the file to be removed.',), -('RemoveFile','InstallMode','N',None, None, None, None, None, '1;2;3','Installation option, one of iimEnum.',), -('RemoveIniFile','Action','N',None, None, None, None, None, '2;4','The type of modification to be made, one of iifEnum.',), -('RemoveIniFile','Value','Y',None, None, None, None, 'Formatted',None, 'The value to be deleted. The value is required when Action is iifIniRemoveTag',), -('RemoveIniFile','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into the Component table referencing component that controls the deletion of the .INI value.',), -('RemoveIniFile','FileName','N',None, None, None, None, 'Filename',None, 'The .INI file name in which to delete the information',), -('RemoveIniFile','DirProperty','Y',None, None, None, None, 'Identifier',None, 'Foreign key into the Directory table denoting the directory where the .INI file is.',), -('RemoveIniFile','Key','N',None, None, None, None, 'Formatted',None, 'The .INI file key below Section.',), -('RemoveIniFile','Section','N',None, None, None, None, 'Formatted',None, 'The .INI file Section.',), -('RemoveIniFile','RemoveIniFile','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token.',), -('RemoveRegistry','Name','Y',None, None, None, None, 'Formatted',None, 'The registry value name.',), -('RemoveRegistry','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into the Component table referencing component that controls the deletion of the registry value.',), -('RemoveRegistry','Key','N',None, None, None, None, 'RegPath',None, 'The key for the registry value.',), -('RemoveRegistry','Root','N',-1,3,None, None, None, None, 'The predefined root key for the registry value, one of rrkEnum',), -('RemoveRegistry','RemoveRegistry','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token.',), -('ReserveCost','Component_','N',None, None, 'Component',1,'Identifier',None, 'Reserve a specified amount of space if this component is to be installed.',), -('ReserveCost','ReserveFolder','Y',None, None, None, None, 'Identifier',None, 'Name of a property whose value is assumed to resolve to the full path to the destination directory',), -('ReserveCost','ReserveKey','N',None, None, None, None, 'Identifier',None, 'Primary key that uniquely identifies a particular ReserveCost record',), -('ReserveCost','ReserveLocal','N',0,2147483647,None, None, None, None, 'Disk space to reserve if linked component is installed locally.',), -('ReserveCost','ReserveSource','N',0,2147483647,None, None, None, None, 'Disk space to reserve if linked component is installed to run from the source location.',), -('SelfReg','File_','N',None, None, 'File',1,'Identifier',None, 'Foreign key into the File table denoting the module that needs to be registered.',), -('SelfReg','Cost','Y',0,32767,None, None, None, None, 'The cost of registering the module.',), -('ServiceControl','Name','N',None, None, None, None, 'Formatted',None, 'Name of a service. /, \\, comma and space are invalid',), -('ServiceControl','Component_','N',None, None, 'Component',1,'Identifier',None, 'Required foreign key into the Component Table that controls the startup of the service',), -('ServiceControl','Event','N',0,187,None, None, None, None, 'Bit field: Install: 0x1 = Start, 0x2 = Stop, 0x8 = Delete, Uninstall: 0x10 = Start, 0x20 = Stop, 0x80 = Delete',), -('ServiceControl','ServiceControl','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token.',), -('ServiceControl','Arguments','Y',None, None, None, None, 'Formatted',None, 'Arguments for the service. Separate by [~].',), -('ServiceControl','Wait','Y',0,1,None, None, None, None, 'Boolean for whether to wait for the service to fully start',), -('ServiceInstall','Name','N',None, None, None, None, 'Formatted',None, 'Internal Name of the Service',), -('ServiceInstall','Description','Y',None, None, None, None, 'Text',None, 'Description of service.',), -('ServiceInstall','Component_','N',None, None, 'Component',1,'Identifier',None, 'Required foreign key into the Component Table that controls the startup of the service',), -('ServiceInstall','Arguments','Y',None, None, None, None, 'Formatted',None, 'Arguments to include in every start of the service, passed to WinMain',), -('ServiceInstall','ServiceInstall','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token.',), -('ServiceInstall','Dependencies','Y',None, None, None, None, 'Formatted',None, 'Other services this depends on to start. Separate by [~], and end with [~][~]',), -('ServiceInstall','DisplayName','Y',None, None, None, None, 'Formatted',None, 'External Name of the Service',), -('ServiceInstall','ErrorControl','N',-2147483647,2147483647,None, None, None, None, 'Severity of error if service fails to start',), -('ServiceInstall','LoadOrderGroup','Y',None, None, None, None, 'Formatted',None, 'LoadOrderGroup',), -('ServiceInstall','Password','Y',None, None, None, None, 'Formatted',None, 'password to run service with. (with StartName)',), -('ServiceInstall','ServiceType','N',-2147483647,2147483647,None, None, None, None, 'Type of the service',), -('ServiceInstall','StartName','Y',None, None, None, None, 'Formatted',None, 'User or object name to run service as',), -('ServiceInstall','StartType','N',0,4,None, None, None, None, 'Type of the service',), -('Shortcut','Name','N',None, None, None, None, 'Filename',None, 'The name of the shortcut to be created.',), -('Shortcut','Description','Y',None, None, None, None, 'Text',None, 'The description for the shortcut.',), -('Shortcut','Component_','N',None, None, 'Component',1,'Identifier',None, 'Foreign key into the Component table denoting the component whose selection gates the shortcut creation/deletion.',), -('Shortcut','Icon_','Y',None, None, 'Icon',1,'Identifier',None, 'Foreign key into the File table denoting the external icon file for the shortcut.',), -('Shortcut','IconIndex','Y',-32767,32767,None, None, None, None, 'The icon index for the shortcut.',), -('Shortcut','Directory_','N',None, None, 'Directory',1,'Identifier',None, 'Foreign key into the Directory table denoting the directory where the shortcut file is created.',), -('Shortcut','Target','N',None, None, None, None, 'Shortcut',None, 'The shortcut target. This is usually a property that is expanded to a file or a folder that the shortcut points to.',), -('Shortcut','Arguments','Y',None, None, None, None, 'Formatted',None, 'The command-line arguments for the shortcut.',), -('Shortcut','Shortcut','N',None, None, None, None, 'Identifier',None, 'Primary key, non-localized token.',), -('Shortcut','Hotkey','Y',0,32767,None, None, None, None, 'The hotkey for the shortcut. It has the virtual-key code for the key in the low-order byte, and the modifier flags in the high-order byte. ',), -('Shortcut','ShowCmd','Y',None, None, None, None, None, '1;3;7','The show command for the application window.The following values may be used.',), -('Shortcut','WkDir','Y',None, None, None, None, 'Identifier',None, 'Name of property defining location of working directory.',), -('Signature','FileName','N',None, None, None, None, 'Filename',None, 'The name of the file. This may contain a "short name|long name" pair.',), -('Signature','Signature','N',None, None, None, None, 'Identifier',None, 'The table key. The Signature represents a unique file signature.',), -('Signature','Languages','Y',None, None, None, None, 'Language',None, 'The languages supported by the file.',), -('Signature','MaxDate','Y',0,2147483647,None, None, None, None, 'The maximum creation date of the file.',), -('Signature','MaxSize','Y',0,2147483647,None, None, None, None, 'The maximum size of the file. ',), -('Signature','MaxVersion','Y',None, None, None, None, 'Text',None, 'The maximum version of the file.',), -('Signature','MinDate','Y',0,2147483647,None, None, None, None, 'The minimum creation date of the file.',), -('Signature','MinSize','Y',0,2147483647,None, None, None, None, 'The minimum size of the file.',), -('Signature','MinVersion','Y',None, None, None, None, 'Text',None, 'The minimum version of the file.',), -('TextStyle','TextStyle','N',None, None, None, None, 'Identifier',None, 'Name of the style. The primary key of this table. This name is embedded in the texts to indicate a style change.',), -('TextStyle','Color','Y',0,16777215,None, None, None, None, 'An integer indicating the color of the string in the RGB format (Red, Green, Blue each 0-255, RGB = R + 256*G + 256^2*B).',), -('TextStyle','FaceName','N',None, None, None, None, 'Text',None, 'A string indicating the name of the font used. Required. The string must be at most 31 characters long.',), -('TextStyle','Size','N',0,32767,None, None, None, None, 'The size of the font used. This size is given in our units (1/12 of the system font height). Assuming that the system font is set to 12 point size, this is equivalent to the point size.',), -('TextStyle','StyleBits','Y',0,15,None, None, None, None, 'A combination of style bits.',), -('TypeLib','Description','Y',None, None, None, None, 'Text',None, None, ), -('TypeLib','Feature_','N',None, None, 'Feature',1,'Identifier',None, 'Required foreign key into the Feature Table, specifying the feature to validate or install in order for the type library to be operational.',), -('TypeLib','Component_','N',None, None, 'Component',1,'Identifier',None, 'Required foreign key into the Component Table, specifying the component for which to return a path when called through LocateComponent.',), -('TypeLib','Directory_','Y',None, None, 'Directory',1,'Identifier',None, 'Optional. The foreign key into the Directory table denoting the path to the help file for the type library.',), -('TypeLib','Language','N',0,32767,None, None, None, None, 'The language of the library.',), -('TypeLib','Version','Y',0,16777215,None, None, None, None, 'The version of the library. The minor version is in the lower 8 bits of the integer. The major version is in the next 16 bits. ',), -('TypeLib','Cost','Y',0,2147483647,None, None, None, None, 'The cost associated with the registration of the typelib. This column is currently optional.',), -('TypeLib','LibID','N',None, None, None, None, 'Guid',None, 'The GUID that represents the library.',), -('UIText','Text','Y',None, None, None, None, 'Text',None, 'The localized version of the string.',), -('UIText','Key','N',None, None, None, None, 'Identifier',None, 'A unique key that identifies the particular string.',), -('Upgrade','Attributes','N',0,2147483647,None, None, None, None, 'The attributes of this product set.',), -('Upgrade','Language','Y',None, None, None, None, 'Language',None, 'A comma-separated list of languages for either products in this set or products not in this set.',), -('Upgrade','ActionProperty','N',None, None, None, None, 'UpperCase',None, 'The property to set when a product in this set is found.',), -('Upgrade','Remove','Y',None, None, None, None, 'Formatted',None, 'The list of features to remove when uninstalling a product from this set. The default is "ALL".',), -('Upgrade','UpgradeCode','N',None, None, None, None, 'Guid',None, 'The UpgradeCode GUID belonging to the products in this set.',), -('Upgrade','VersionMax','Y',None, None, None, None, 'Text',None, 'The maximum ProductVersion of the products in this set. The set may or may not include products with this particular version.',), -('Upgrade','VersionMin','Y',None, None, None, None, 'Text',None, 'The minimum ProductVersion of the products in this set. The set may or may not include products with this particular version.',), -('Verb','Sequence','Y',0,32767,None, None, None, None, 'Order within the verbs for a particular extension. Also used simply to specify the default verb.',), -('Verb','Argument','Y',None, None, None, None, 'Formatted',None, 'Optional value for the command arguments.',), -('Verb','Extension_','N',None, None, 'Extension',1,'Text',None, 'The extension associated with the table row.',), -('Verb','Verb','N',None, None, None, None, 'Text',None, 'The verb for the command.',), -('Verb','Command','Y',None, None, None, None, 'Formatted',None, 'The command text.',), -] diff --git a/Lib/msilib/sequence.py b/Lib/msilib/sequence.py deleted file mode 100644 index 7012a21f857219..00000000000000 --- a/Lib/msilib/sequence.py +++ /dev/null @@ -1,126 +0,0 @@ -AdminExecuteSequence = [ -('InstallInitialize', None, 1500), -('InstallFinalize', None, 6600), -('InstallFiles', None, 4000), -('InstallAdminPackage', None, 3900), -('FileCost', None, 900), -('CostInitialize', None, 800), -('CostFinalize', None, 1000), -('InstallValidate', None, 1400), -] - -AdminUISequence = [ -('FileCost', None, 900), -('CostInitialize', None, 800), -('CostFinalize', None, 1000), -('ExecuteAction', None, 1300), -('ExitDialog', None, -1), -('FatalError', None, -3), -('UserExit', None, -2), -] - -AdvtExecuteSequence = [ -('InstallInitialize', None, 1500), -('InstallFinalize', None, 6600), -('CostInitialize', None, 800), -('CostFinalize', None, 1000), -('InstallValidate', None, 1400), -('CreateShortcuts', None, 4500), -('MsiPublishAssemblies', None, 6250), -('PublishComponents', None, 6200), -('PublishFeatures', None, 6300), -('PublishProduct', None, 6400), -('RegisterClassInfo', None, 4600), -('RegisterExtensionInfo', None, 4700), -('RegisterMIMEInfo', None, 4900), -('RegisterProgIdInfo', None, 4800), -] - -InstallExecuteSequence = [ -('InstallInitialize', None, 1500), -('InstallFinalize', None, 6600), -('InstallFiles', None, 4000), -('FileCost', None, 900), -('CostInitialize', None, 800), -('CostFinalize', None, 1000), -('InstallValidate', None, 1400), -('CreateShortcuts', None, 4500), -('MsiPublishAssemblies', None, 6250), -('PublishComponents', None, 6200), -('PublishFeatures', None, 6300), -('PublishProduct', None, 6400), -('RegisterClassInfo', None, 4600), -('RegisterExtensionInfo', None, 4700), -('RegisterMIMEInfo', None, 4900), -('RegisterProgIdInfo', None, 4800), -('AllocateRegistrySpace', 'NOT Installed', 1550), -('AppSearch', None, 400), -('BindImage', None, 4300), -('CCPSearch', 'NOT Installed', 500), -('CreateFolders', None, 3700), -('DeleteServices', 'VersionNT', 2000), -('DuplicateFiles', None, 4210), -('FindRelatedProducts', None, 200), -('InstallODBC', None, 5400), -('InstallServices', 'VersionNT', 5800), -('IsolateComponents', None, 950), -('LaunchConditions', None, 100), -('MigrateFeatureStates', None, 1200), -('MoveFiles', None, 3800), -('PatchFiles', None, 4090), -('ProcessComponents', None, 1600), -('RegisterComPlus', None, 5700), -('RegisterFonts', None, 5300), -('RegisterProduct', None, 6100), -('RegisterTypeLibraries', None, 5500), -('RegisterUser', None, 6000), -('RemoveDuplicateFiles', None, 3400), -('RemoveEnvironmentStrings', None, 3300), -('RemoveExistingProducts', None, 6700), -('RemoveFiles', None, 3500), -('RemoveFolders', None, 3600), -('RemoveIniValues', None, 3100), -('RemoveODBC', None, 2400), -('RemoveRegistryValues', None, 2600), -('RemoveShortcuts', None, 3200), -('RMCCPSearch', 'NOT Installed', 600), -('SelfRegModules', None, 5600), -('SelfUnregModules', None, 2200), -('SetODBCFolders', None, 1100), -('StartServices', 'VersionNT', 5900), -('StopServices', 'VersionNT', 1900), -('MsiUnpublishAssemblies', None, 1750), -('UnpublishComponents', None, 1700), -('UnpublishFeatures', None, 1800), -('UnregisterClassInfo', None, 2700), -('UnregisterComPlus', None, 2100), -('UnregisterExtensionInfo', None, 2800), -('UnregisterFonts', None, 2500), -('UnregisterMIMEInfo', None, 3000), -('UnregisterProgIdInfo', None, 2900), -('UnregisterTypeLibraries', None, 2300), -('ValidateProductID', None, 700), -('WriteEnvironmentStrings', None, 5200), -('WriteIniValues', None, 5100), -('WriteRegistryValues', None, 5000), -] - -InstallUISequence = [ -('FileCost', None, 900), -('CostInitialize', None, 800), -('CostFinalize', None, 1000), -('ExecuteAction', None, 1300), -('ExitDialog', None, -1), -('FatalError', None, -3), -('UserExit', None, -2), -('AppSearch', None, 400), -('CCPSearch', 'NOT Installed', 500), -('FindRelatedProducts', None, 200), -('IsolateComponents', None, 950), -('LaunchConditions', None, 100), -('MigrateFeatureStates', None, 1200), -('RMCCPSearch', 'NOT Installed', 600), -('ValidateProductID', None, 700), -] - -tables=['AdminExecuteSequence', 'AdminUISequence', 'AdvtExecuteSequence', 'InstallExecuteSequence', 'InstallUISequence'] diff --git a/Lib/msilib/text.py b/Lib/msilib/text.py deleted file mode 100644 index c4b9e8697d728d..00000000000000 --- a/Lib/msilib/text.py +++ /dev/null @@ -1,129 +0,0 @@ -import msilib,os;dirname=os.path.dirname(__file__) - -ActionText = [ -('InstallValidate', 'Validating install', None), -('InstallFiles', 'Copying new files', 'File: [1], Directory: [9], Size: [6]'), -('InstallAdminPackage', 'Copying network install files', 'File: [1], Directory: [9], Size: [6]'), -('FileCost', 'Computing space requirements', None), -('CostInitialize', 'Computing space requirements', None), -('CostFinalize', 'Computing space requirements', None), -('CreateShortcuts', 'Creating shortcuts', 'Shortcut: [1]'), -('PublishComponents', 'Publishing Qualified Components', 'Component ID: [1], Qualifier: [2]'), -('PublishFeatures', 'Publishing Product Features', 'Feature: [1]'), -('PublishProduct', 'Publishing product information', None), -('RegisterClassInfo', 'Registering Class servers', 'Class Id: [1]'), -('RegisterExtensionInfo', 'Registering extension servers', 'Extension: [1]'), -('RegisterMIMEInfo', 'Registering MIME info', 'MIME Content Type: [1], Extension: [2]'), -('RegisterProgIdInfo', 'Registering program identifiers', 'ProgId: [1]'), -('AllocateRegistrySpace', 'Allocating registry space', 'Free space: [1]'), -('AppSearch', 'Searching for installed applications', 'Property: [1], Signature: [2]'), -('BindImage', 'Binding executables', 'File: [1]'), -('CCPSearch', 'Searching for qualifying products', None), -('CreateFolders', 'Creating folders', 'Folder: [1]'), -('DeleteServices', 'Deleting services', 'Service: [1]'), -('DuplicateFiles', 'Creating duplicate files', 'File: [1], Directory: [9], Size: [6]'), -('FindRelatedProducts', 'Searching for related applications', 'Found application: [1]'), -('InstallODBC', 'Installing ODBC components', None), -('InstallServices', 'Installing new services', 'Service: [2]'), -('LaunchConditions', 'Evaluating launch conditions', None), -('MigrateFeatureStates', 'Migrating feature states from related applications', 'Application: [1]'), -('MoveFiles', 'Moving files', 'File: [1], Directory: [9], Size: [6]'), -('PatchFiles', 'Patching files', 'File: [1], Directory: [2], Size: [3]'), -('ProcessComponents', 'Updating component registration', None), -('RegisterComPlus', 'Registering COM+ Applications and Components', 'AppId: [1]{{, AppType: [2], Users: [3], RSN: [4]}}'), -('RegisterFonts', 'Registering fonts', 'Font: [1]'), -('RegisterProduct', 'Registering product', '[1]'), -('RegisterTypeLibraries', 'Registering type libraries', 'LibID: [1]'), -('RegisterUser', 'Registering user', '[1]'), -('RemoveDuplicateFiles', 'Removing duplicated files', 'File: [1], Directory: [9]'), -('RemoveEnvironmentStrings', 'Updating environment strings', 'Name: [1], Value: [2], Action [3]'), -('RemoveExistingProducts', 'Removing applications', 'Application: [1], Command line: [2]'), -('RemoveFiles', 'Removing files', 'File: [1], Directory: [9]'), -('RemoveFolders', 'Removing folders', 'Folder: [1]'), -('RemoveIniValues', 'Removing INI files entries', 'File: [1], Section: [2], Key: [3], Value: [4]'), -('RemoveODBC', 'Removing ODBC components', None), -('RemoveRegistryValues', 'Removing system registry values', 'Key: [1], Name: [2]'), -('RemoveShortcuts', 'Removing shortcuts', 'Shortcut: [1]'), -('RMCCPSearch', 'Searching for qualifying products', None), -('SelfRegModules', 'Registering modules', 'File: [1], Folder: [2]'), -('SelfUnregModules', 'Unregistering modules', 'File: [1], Folder: [2]'), -('SetODBCFolders', 'Initializing ODBC directories', None), -('StartServices', 'Starting services', 'Service: [1]'), -('StopServices', 'Stopping services', 'Service: [1]'), -('UnpublishComponents', 'Unpublishing Qualified Components', 'Component ID: [1], Qualifier: [2]'), -('UnpublishFeatures', 'Unpublishing Product Features', 'Feature: [1]'), -('UnregisterClassInfo', 'Unregister Class servers', 'Class Id: [1]'), -('UnregisterComPlus', 'Unregistering COM+ Applications and Components', 'AppId: [1]{{, AppType: [2]}}'), -('UnregisterExtensionInfo', 'Unregistering extension servers', 'Extension: [1]'), -('UnregisterFonts', 'Unregistering fonts', 'Font: [1]'), -('UnregisterMIMEInfo', 'Unregistering MIME info', 'MIME Content Type: [1], Extension: [2]'), -('UnregisterProgIdInfo', 'Unregistering program identifiers', 'ProgId: [1]'), -('UnregisterTypeLibraries', 'Unregistering type libraries', 'LibID: [1]'), -('WriteEnvironmentStrings', 'Updating environment strings', 'Name: [1], Value: [2], Action [3]'), -('WriteIniValues', 'Writing INI files values', 'File: [1], Section: [2], Key: [3], Value: [4]'), -('WriteRegistryValues', 'Writing system registry values', 'Key: [1], Name: [2], Value: [3]'), -('Advertise', 'Advertising application', None), -('GenerateScript', 'Generating script operations for action:', '[1]'), -('InstallSFPCatalogFile', 'Installing system catalog', 'File: [1], Dependencies: [2]'), -('MsiPublishAssemblies', 'Publishing assembly information', 'Application Context:[1], Assembly Name:[2]'), -('MsiUnpublishAssemblies', 'Unpublishing assembly information', 'Application Context:[1], Assembly Name:[2]'), -('Rollback', 'Rolling back action:', '[1]'), -('RollbackCleanup', 'Removing backup files', 'File: [1]'), -('UnmoveFiles', 'Removing moved files', 'File: [1], Directory: [9]'), -('UnpublishProduct', 'Unpublishing product information', None), -] - -UIText = [ -('AbsentPath', None), -('bytes', 'bytes'), -('GB', 'GB'), -('KB', 'KB'), -('MB', 'MB'), -('MenuAbsent', 'Entire feature will be unavailable'), -('MenuAdvertise', 'Feature will be installed when required'), -('MenuAllCD', 'Entire feature will be installed to run from CD'), -('MenuAllLocal', 'Entire feature will be installed on local hard drive'), -('MenuAllNetwork', 'Entire feature will be installed to run from network'), -('MenuCD', 'Will be installed to run from CD'), -('MenuLocal', 'Will be installed on local hard drive'), -('MenuNetwork', 'Will be installed to run from network'), -('ScriptInProgress', 'Gathering required information...'), -('SelAbsentAbsent', 'This feature will remain uninstalled'), -('SelAbsentAdvertise', 'This feature will be set to be installed when required'), -('SelAbsentCD', 'This feature will be installed to run from CD'), -('SelAbsentLocal', 'This feature will be installed on the local hard drive'), -('SelAbsentNetwork', 'This feature will be installed to run from the network'), -('SelAdvertiseAbsent', 'This feature will become unavailable'), -('SelAdvertiseAdvertise', 'Will be installed when required'), -('SelAdvertiseCD', 'This feature will be available to run from CD'), -('SelAdvertiseLocal', 'This feature will be installed on your local hard drive'), -('SelAdvertiseNetwork', 'This feature will be available to run from the network'), -('SelCDAbsent', "This feature will be uninstalled completely, you won't be able to run it from CD"), -('SelCDAdvertise', 'This feature will change from run from CD state to set to be installed when required'), -('SelCDCD', 'This feature will remain to be run from CD'), -('SelCDLocal', 'This feature will change from run from CD state to be installed on the local hard drive'), -('SelChildCostNeg', 'This feature frees up [1] on your hard drive.'), -('SelChildCostPos', 'This feature requires [1] on your hard drive.'), -('SelCostPending', 'Compiling cost for this feature...'), -('SelLocalAbsent', 'This feature will be completely removed'), -('SelLocalAdvertise', 'This feature will be removed from your local hard drive, but will be set to be installed when required'), -('SelLocalCD', 'This feature will be removed from your local hard drive, but will be still available to run from CD'), -('SelLocalLocal', 'This feature will remain on you local hard drive'), -('SelLocalNetwork', 'This feature will be removed from your local hard drive, but will be still available to run from the network'), -('SelNetworkAbsent', "This feature will be uninstalled completely, you won't be able to run it from the network"), -('SelNetworkAdvertise', 'This feature will change from run from network state to set to be installed when required'), -('SelNetworkLocal', 'This feature will change from run from network state to be installed on the local hard drive'), -('SelNetworkNetwork', 'This feature will remain to be run from the network'), -('SelParentCostNegNeg', 'This feature frees up [1] on your hard drive. It has [2] of [3] subfeatures selected. The subfeatures free up [4] on your hard drive.'), -('SelParentCostNegPos', 'This feature frees up [1] on your hard drive. It has [2] of [3] subfeatures selected. The subfeatures require [4] on your hard drive.'), -('SelParentCostPosNeg', 'This feature requires [1] on your hard drive. It has [2] of [3] subfeatures selected. The subfeatures free up [4] on your hard drive.'), -('SelParentCostPosPos', 'This feature requires [1] on your hard drive. It has [2] of [3] subfeatures selected. The subfeatures require [4] on your hard drive.'), -('TimeRemaining', 'Time remaining: {[1] minutes }{[2] seconds}'), -('VolumeCostAvailable', 'Available'), -('VolumeCostDifference', 'Difference'), -('VolumeCostRequired', 'Required'), -('VolumeCostSize', 'Disk Size'), -('VolumeCostVolume', 'Volume'), -] - -tables=['ActionText', 'UIText'] diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 1a8822b9db012d..dbbf106f680964 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -9,6 +9,7 @@ __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ] +import errno import io import os import sys @@ -271,12 +272,22 @@ class PipeConnection(_ConnectionBase): with FILE_FLAG_OVERLAPPED. """ _got_empty_message = False + _send_ov = None def _close(self, _CloseHandle=_winapi.CloseHandle): + ov = self._send_ov + if ov is not None: + # Interrupt WaitForMultipleObjects() in _send_bytes() + ov.cancel() _CloseHandle(self._handle) def _send_bytes(self, buf): + if self._send_ov is not None: + # A connection should only be used by a single thread + raise ValueError("concurrent send_bytes() calls " + "are not supported") ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True) + self._send_ov = ov try: if err == _winapi.ERROR_IO_PENDING: waitres = _winapi.WaitForMultipleObjects( @@ -286,7 +297,13 @@ def _send_bytes(self, buf): ov.cancel() raise finally: + self._send_ov = None nwritten, err = ov.GetOverlappedResult(True) + if err == _winapi.ERROR_OPERATION_ABORTED: + # close() was called by another thread while + # WaitForMultipleObjects() was waiting for the overlapped + # operation. + raise OSError(errno.EPIPE, "handle is closed") assert err == 0 assert nwritten == len(buf) @@ -722,11 +739,11 @@ def PipeClient(address): # Authentication stuff # -MESSAGE_LENGTH = 20 +MESSAGE_LENGTH = 40 # MUST be > 20 -CHALLENGE = b'#CHALLENGE#' -WELCOME = b'#WELCOME#' -FAILURE = b'#FAILURE#' +_CHALLENGE = b'#CHALLENGE#' +_WELCOME = b'#WELCOME#' +_FAILURE = b'#FAILURE#' # multiprocessing.connection Authentication Handshake Protocol Description # (as documented for reference after reading the existing code) @@ -750,7 +767,12 @@ def PipeClient(address): # ------------------------------ --------------------------------------- # 0. Open a connection on the pipe. # 1. Accept connection. -# 2. New random 20 bytes -> MESSAGE +# 2. Random 20+ bytes -> MESSAGE +# Modern servers always send +# more than 20 bytes and include +# a {digest} prefix on it with +# their preferred HMAC digest. +# Legacy ones send ==20 bytes. # 3. send 4 byte length (net order) # prefix followed by: # b'#CHALLENGE#' + MESSAGE @@ -763,14 +785,32 @@ def PipeClient(address): # 6. Assert that M1 starts with: # b'#CHALLENGE#' # 7. Strip that prefix from M1 into -> M2 -# 8. Compute HMAC-MD5 of AUTHKEY, M2 -> C_DIGEST +# 7.1. Parse M2: if it is exactly 20 bytes in +# length this indicates a legacy server +# supporting only HMAC-MD5. Otherwise the +# 7.2. preferred digest is looked up from an +# expected "{digest}" prefix on M2. No prefix +# or unsupported digest? <- AuthenticationError +# 7.3. Put divined algorithm name in -> D_NAME +# 8. Compute HMAC-D_NAME of AUTHKEY, M2 -> C_DIGEST # 9. Send 4 byte length prefix (net order) # followed by C_DIGEST bytes. -# 10. Compute HMAC-MD5 of AUTHKEY, -# MESSAGE into -> M_DIGEST. -# 11. Receive 4 or 4+8 byte length +# 10. Receive 4 or 4+8 byte length # prefix (#4 dance) -> SIZE. -# 12. Receive min(SIZE, 256) -> C_D. +# 11. Receive min(SIZE, 256) -> C_D. +# 11.1. Parse C_D: legacy servers +# accept it as is, "md5" -> D_NAME +# 11.2. modern servers check the length +# of C_D, IF it is 16 bytes? +# 11.2.1. "md5" -> D_NAME +# and skip to step 12. +# 11.3. longer? expect and parse a "{digest}" +# prefix into -> D_NAME. +# Strip the prefix and store remaining +# bytes in -> C_D. +# 11.4. Don't like D_NAME? <- AuthenticationError +# 12. Compute HMAC-D_NAME of AUTHKEY, +# MESSAGE into -> M_DIGEST. # 13. Compare M_DIGEST == C_D: # 14a: Match? Send length prefix & # b'#WELCOME#' @@ -787,42 +827,139 @@ def PipeClient(address): # # If this RETURNed, the connection remains open: it has been authenticated. # -# Length prefixes are used consistently even though every step so far has -# always been a singular specific fixed length. This may help us evolve -# the protocol in the future without breaking backwards compatibility. -# -# Similarly the initial challenge message from the serving side has always -# been 20 bytes, but clients can accept a 100+ so using the length of the -# opening challenge message as an indicator of protocol version may work. +# Length prefixes are used consistently. Even on the legacy protocol, this +# was good fortune and allowed us to evolve the protocol by using the length +# of the opening challenge or length of the returned digest as a signal as +# to which protocol the other end supports. + +_ALLOWED_DIGESTS = frozenset( + {b'md5', b'sha256', b'sha384', b'sha3_256', b'sha3_384'}) +_MAX_DIGEST_LEN = max(len(_) for _ in _ALLOWED_DIGESTS) + +# Old hmac-md5 only server versions from Python <=3.11 sent a message of this +# length. It happens to not match the length of any supported digest so we can +# use a message of this length to indicate that we should work in backwards +# compatible md5-only mode without a {digest_name} prefix on our response. +_MD5ONLY_MESSAGE_LENGTH = 20 +_MD5_DIGEST_LEN = 16 +_LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN) + + +def _get_digest_name_and_payload(message: bytes) -> (str, bytes): + """Returns a digest name and the payload for a response hash. + + If a legacy protocol is detected based on the message length + or contents the digest name returned will be empty to indicate + legacy mode where MD5 and no digest prefix should be sent. + """ + # modern message format: b"{digest}payload" longer than 20 bytes + # legacy message format: 16 or 20 byte b"payload" + if len(message) in _LEGACY_LENGTHS: + # Either this was a legacy server challenge, or we're processing + # a reply from a legacy client that sent an unprefixed 16-byte + # HMAC-MD5 response. All messages using the modern protocol will + # be longer than either of these lengths. + return '', message + if (message.startswith(b'{') and + (curly := message.find(b'}', 1, _MAX_DIGEST_LEN+2)) > 0): + digest = message[1:curly] + if digest in _ALLOWED_DIGESTS: + payload = message[curly+1:] + return digest.decode('ascii'), payload + raise AuthenticationError( + 'unsupported message length, missing digest prefix, ' + f'or unsupported digest: {message=}') + + +def _create_response(authkey, message): + """Create a MAC based on authkey and message + + The MAC algorithm defaults to HMAC-MD5, unless MD5 is not available or + the message has a '{digest_name}' prefix. For legacy HMAC-MD5, the response + is the raw MAC, otherwise the response is prefixed with '{digest_name}', + e.g. b'{sha256}abcdefg...' + + Note: The MAC protects the entire message including the digest_name prefix. + """ + import hmac + digest_name = _get_digest_name_and_payload(message)[0] + # The MAC protects the entire message: digest header and payload. + if not digest_name: + # Legacy server without a {digest} prefix on message. + # Generate a legacy non-prefixed HMAC-MD5 reply. + try: + return hmac.new(authkey, message, 'md5').digest() + except ValueError: + # HMAC-MD5 is not available (FIPS mode?), fall back to + # HMAC-SHA2-256 modern protocol. The legacy server probably + # doesn't support it and will reject us anyways. :shrug: + digest_name = 'sha256' + # Modern protocol, indicate the digest used in the reply. + response = hmac.new(authkey, message, digest_name).digest() + return b'{%s}%s' % (digest_name.encode('ascii'), response) + + +def _verify_challenge(authkey, message, response): + """Verify MAC challenge + + If our message did not include a digest_name prefix, the client is allowed + to select a stronger digest_name from _ALLOWED_DIGESTS. + + In case our message is prefixed, a client cannot downgrade to a weaker + algorithm, because the MAC is calculated over the entire message + including the '{digest_name}' prefix. + """ + import hmac + response_digest, response_mac = _get_digest_name_and_payload(response) + response_digest = response_digest or 'md5' + try: + expected = hmac.new(authkey, message, response_digest).digest() + except ValueError: + raise AuthenticationError(f'{response_digest=} unsupported') + if len(expected) != len(response_mac): + raise AuthenticationError( + f'expected {response_digest!r} of length {len(expected)} ' + f'got {len(response_mac)}') + if not hmac.compare_digest(expected, response_mac): + raise AuthenticationError('digest received was wrong') -def deliver_challenge(connection, authkey): - import hmac +def deliver_challenge(connection, authkey: bytes, digest_name='sha256'): if not isinstance(authkey, bytes): raise ValueError( "Authkey must be bytes, not {0!s}".format(type(authkey))) + assert MESSAGE_LENGTH > _MD5ONLY_MESSAGE_LENGTH, "protocol constraint" message = os.urandom(MESSAGE_LENGTH) - connection.send_bytes(CHALLENGE + message) - digest = hmac.new(authkey, message, 'md5').digest() + message = b'{%s}%s' % (digest_name.encode('ascii'), message) + # Even when sending a challenge to a legacy client that does not support + # digest prefixes, they'll take the entire thing as a challenge and + # respond to it with a raw HMAC-MD5. + connection.send_bytes(_CHALLENGE + message) response = connection.recv_bytes(256) # reject large message - if response == digest: - connection.send_bytes(WELCOME) + try: + _verify_challenge(authkey, message, response) + except AuthenticationError: + connection.send_bytes(_FAILURE) + raise else: - connection.send_bytes(FAILURE) - raise AuthenticationError('digest received was wrong') + connection.send_bytes(_WELCOME) -def answer_challenge(connection, authkey): - import hmac + +def answer_challenge(connection, authkey: bytes): if not isinstance(authkey, bytes): raise ValueError( "Authkey must be bytes, not {0!s}".format(type(authkey))) message = connection.recv_bytes(256) # reject large message - assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message - message = message[len(CHALLENGE):] - digest = hmac.new(authkey, message, 'md5').digest() + if not message.startswith(_CHALLENGE): + raise AuthenticationError( + f'Protocol error, expected challenge: {message=}') + message = message[len(_CHALLENGE):] + if len(message) < _MD5ONLY_MESSAGE_LENGTH: + raise AuthenticationError('challenge too short: {len(message)} bytes') + digest = _create_response(authkey, message) connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message - if response != WELCOME: + if response != _WELCOME: raise AuthenticationError('digest sent was rejected') # diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 22a911a7a29cdc..4642707dae2f4e 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -61,7 +61,7 @@ def _stop_unlocked(self): def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' - if not all(type(mod) is str for mod in self._preload_modules): + if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') self._preload_modules = modules_names diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index b6534939b4d98b..96cebc6eabec89 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -90,7 +90,10 @@ def dispatch(c, id, methodname, args=(), kwds={}): kind, result = c.recv() if kind == '#RETURN': return result - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def convert_to_error(kind, result): if kind == '#ERROR': @@ -833,7 +836,10 @@ def _callmethod(self, methodname, args=(), kwds={}): conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def _getvalue(self): ''' @@ -1159,15 +1165,19 @@ def __imul__(self, value): self._callmethod('__imul__', (value,)) return self + __class_getitem__ = classmethod(types.GenericAlias) + -DictProxy = MakeProxyType('DictProxy', ( +_BaseDictProxy = MakeProxyType('DictProxy', ( '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', '__setitem__', 'clear', 'copy', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) -DictProxy._method_to_typeid_ = { +_BaseDictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } +class DictProxy(_BaseDictProxy): + __class_getitem__ = classmethod(types.GenericAlias) ArrayProxy = MakeProxyType('ArrayProxy', ( diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 4f5d88cb975cb7..f979890170b1a1 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -200,7 +200,7 @@ def __init__(self, processes=None, initializer=None, initargs=(), self._initargs = initargs if processes is None: - processes = os.cpu_count() or 1 + processes = os.process_cpu_count() or 1 if processes < 1: raise ValueError("Number of processes must be at least 1") if maxtasksperchild is not None: diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 4d60ffc030bea6..af044305709e56 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -14,6 +14,7 @@ # # +# Exit code used by Popen.terminate() TERMINATE = 0x10000 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") @@ -122,9 +123,15 @@ def terminate(self): if self.returncode is None: try: _winapi.TerminateProcess(int(self._handle), TERMINATE) - except OSError: - if self.wait(timeout=1.0) is None: + except PermissionError: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + code = _winapi.GetExitCodeProcess(int(self._handle)) + if code == _winapi.STILL_ACTIVE: raise + self.returncode = code + else: + self.returncode = -signal.SIGTERM kill = terminate diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index c03c859baa795b..271ba3fd325138 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -61,7 +61,7 @@ def parent_process(): def _cleanup(): # check for processes which have finished for p in list(_children): - if p._popen.poll() is not None: + if (child_popen := p._popen) and child_popen.poll() is not None: _children.discard(p) # diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index daf9ee94a19431..852ae87b276861 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -158,6 +158,20 @@ def cancel_join_thread(self): except AttributeError: pass + def _terminate_broken(self): + # Close a Queue on error. + + # gh-94777: Prevent queue writing to a pipe which is no longer read. + self._reader.close() + + # gh-107219: Close the connection writer which can unblock + # Queue._feed() if it was stuck in send_bytes(). + if sys.platform == 'win32': + self._writer.close() + + self.close() + self.join_thread() + def _start_thread(self): debug('Queue._start_thread()') @@ -169,13 +183,19 @@ def _start_thread(self): self._wlock, self._reader.close, self._writer.close, self._ignore_epipe, self._on_queue_feeder_error, self._sem), - name='QueueFeederThread' + name='QueueFeederThread', + daemon=True, ) - self._thread.daemon = True - debug('doing self._thread.start()') - self._thread.start() - debug('... done self._thread.start()') + try: + debug('doing self._thread.start()') + self._thread.start() + debug('... done self._thread.start()') + except: + # gh-109047: During Python finalization, creating a thread + # can fail with RuntimeError. + self._thread = None + raise if not self._joincancelled: self._jointhread = Finalize( diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index ea369507297f86..8e41f461cc934e 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -51,15 +51,31 @@ }) +class ReentrantCallError(RuntimeError): + pass + + class ResourceTracker(object): def __init__(self): - self._lock = threading.Lock() + self._lock = threading.RLock() self._fd = None self._pid = None + def _reentrant_call_error(self): + # gh-109629: this happens if an explicit call to the ResourceTracker + # gets interrupted by a garbage collection, invoking a finalizer (*) + # that itself calls back into ResourceTracker. + # (*) for example the SemLock finalizer + raise ReentrantCallError( + "Reentrant call into the multiprocessing resource tracker") + def _stop(self): with self._lock: + # This should not happen (_stop() isn't called by a finalizer) + # but we check for it anyway. + if self._lock._recursion_count() > 1: + return self._reentrant_call_error() if self._fd is None: # not running return @@ -81,6 +97,9 @@ def ensure_running(self): This can be run from any process. Usually a child process will use the resource created by its parent.''' with self._lock: + if self._lock._recursion_count() > 1: + # The code below is certainly not reentrant-safe, so bail out + return self._reentrant_call_error() if self._fd is not None: # resource tracker was launched before, is it still running? if self._check_alive(): @@ -159,7 +178,17 @@ def unregister(self, name, rtype): self._send('UNREGISTER', name, rtype) def _send(self, cmd, name, rtype): - self.ensure_running() + try: + self.ensure_running() + except ReentrantCallError: + # The code below might or might not work, depending on whether + # the resource tracker was already running and still alive. + # Better warn the user. + # (XXX is warnings.warn itself reentrant-safe? :-) + warnings.warn( + f"ResourceTracker called reentrantly for resource cleanup, " + f"which is unsupported. " + f"The {rtype} object {name!r} might leak.") msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') if len(msg) > 512: # posix guarantees that writes to a pipe of less than PIPE_BUF @@ -176,6 +205,7 @@ def _send(self, cmd, name, rtype): unregister = _resource_tracker.unregister getfd = _resource_tracker.getfd + def main(fd): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc @@ -221,9 +251,10 @@ def main(fd): for rtype, rtype_cache in cache.items(): if rtype_cache: try: - warnings.warn('resource_tracker: There appear to be %d ' - 'leaked %s objects to clean up at shutdown' % - (len(rtype_cache), rtype)) + warnings.warn( + f'resource_tracker: There appear to be {len(rtype_cache)} ' + f'leaked {rtype} objects to clean up at shutdown: {rtype_cache}' + ) except Exception: pass for name in rtype_cache: diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 9a1e5aa17b87a2..67e70fdc27cf31 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -71,8 +71,9 @@ class SharedMemory: _flags = os.O_RDWR _mode = 0o600 _prepend_leading_slash = True if _USE_POSIX else False + _track = True - def __init__(self, name=None, create=False, size=0): + def __init__(self, name=None, create=False, size=0, *, track=True): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: @@ -82,6 +83,7 @@ def __init__(self, name=None, create=False, size=0): if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") + self._track = track if _USE_POSIX: # POSIX Shared Memory @@ -116,8 +118,8 @@ def __init__(self, name=None, create=False, size=0): except OSError: self.unlink() raise - - resource_tracker.register(self._name, "shared_memory") + if self._track: + resource_tracker.register(self._name, "shared_memory") else: @@ -236,12 +238,20 @@ def close(self): def unlink(self): """Requests that the underlying shared memory block be destroyed. - In order to ensure proper cleanup of resources, unlink should be - called once (and only once) across all processes which have access - to the shared memory block.""" + Unlink should be called once (and only once) across all handles + which have access to the shared memory block, even if these + handles belong to different processes. Closing and unlinking may + happen in any order, but trying to access data inside a shared + memory block after unlinking may result in memory errors, + depending on platform. + + This method has no effect on Windows, where the only way to + delete a shared memory block is to close all handles.""" + if _USE_POSIX and self._name: _posixshmem.shm_unlink(self._name) - resource_tracker.unregister(self._name, "shared_memory") + if self._track: + resource_tracker.unregister(self._name, "shared_memory") _encoding = "utf8" diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 09f8a229d7cccb..daac1ecc34b55e 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -31,11 +31,13 @@ WINSERVICE = False else: WINEXE = getattr(sys, 'frozen', False) - WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") + WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe") def set_executable(exe): global _python_exe - if sys.platform == 'win32': + if exe is None: + _python_exe = exe + elif sys.platform == 'win32': _python_exe = os.fsdecode(exe) else: _python_exe = os.fsencode(exe) @@ -148,7 +150,11 @@ def _check_not_importing_main(): ... The "freeze_support()" line can be omitted if the program - is not going to be frozen to produce an executable.''') + is not going to be frozen to produce an executable. + + To fix this issue, refer to the "Safe importing of main module" + section in https://docs.python.org/3/library/multiprocessing.html + ''') def get_preparation_data(name): diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 42624b543601a1..3ccbfe311c71f3 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -50,8 +50,8 @@ class SemLock(object): def __init__(self, kind, value, maxvalue, *, ctx): if ctx is None: ctx = context._default_context.get_context() - name = ctx.get_start_method() - unlink_now = sys.platform == 'win32' or name == 'fork' + self._is_fork_ctx = ctx.get_start_method() == 'fork' + unlink_now = sys.platform == 'win32' or self._is_fork_ctx for i in range(100): try: sl = self._semlock = _multiprocessing.SemLock( @@ -103,6 +103,11 @@ def __getstate__(self): if sys.platform == 'win32': h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: + if self._is_fork_ctx: + raise RuntimeError('A SemLock created in a fork context is being ' + 'shared with a process in a spawn context. This is ' + 'not supported. Please use the same context to create ' + 'multiprocessing objects and Process.') h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) @@ -110,6 +115,8 @@ def __setstate__(self, state): self._semlock = _multiprocessing.SemLock._rebuild(*state) util.debug('recreated blocker with handle %r' % state[0]) self._make_methods() + # Ensure that deserialized SemLock can be serialized again (gh-108520). + self._is_fork_ctx = False @staticmethod def _make_name(): diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 6ee0d33e88a060..28c77df1c32ea8 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -64,8 +64,7 @@ def get_logger(): global _logger import logging - logging._acquireLock() - try: + with logging._lock: if not _logger: _logger = logging.getLogger(LOGGER_NAME) @@ -79,9 +78,6 @@ def get_logger(): atexit._exithandlers.remove((_exit_function, (), {})) atexit._exithandlers.append((_exit_function, (), {})) - finally: - logging._releaseLock() - return _logger def log_to_stderr(level=None): diff --git a/Lib/nntplib.py b/Lib/nntplib.py deleted file mode 100644 index dddea059982be1..00000000000000 --- a/Lib/nntplib.py +++ /dev/null @@ -1,1093 +0,0 @@ -"""An NNTP client class based on: -- RFC 977: Network News Transfer Protocol -- RFC 2980: Common NNTP Extensions -- RFC 3977: Network News Transfer Protocol (version 2) - -Example: - ->>> from nntplib import NNTP ->>> s = NNTP('news') ->>> resp, count, first, last, name = s.group('comp.lang.python') ->>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) -Group comp.lang.python has 51 articles, range 5770 to 5821 ->>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last)) ->>> resp = s.quit() ->>> - -Here 'resp' is the server response line. -Error responses are turned into exceptions. - -To post an article from a file: ->>> f = open(filename, 'rb') # file containing article, including header ->>> resp = s.post(f) ->>> - -For descriptions of all methods, read the comments in the code below. -Note that all arguments and return values representing article numbers -are strings, not numbers, since they are rarely used for calculations. -""" - -# RFC 977 by Brian Kantor and Phil Lapsley. -# xover, xgtitle, xpath, date methods by Kevan Heydon - -# Incompatible changes from the 2.x nntplib: -# - all commands are encoded as UTF-8 data (using the "surrogateescape" -# error handler), except for raw message data (POST, IHAVE) -# - all responses are decoded as UTF-8 data (using the "surrogateescape" -# error handler), except for raw message data (ARTICLE, HEAD, BODY) -# - the `file` argument to various methods is keyword-only -# -# - NNTP.date() returns a datetime object -# - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object, -# rather than a pair of (date, time) strings. -# - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples -# - NNTP.descriptions() returns a dict mapping group names to descriptions -# - NNTP.xover() returns a list of dicts mapping field names (header or metadata) -# to field values; each dict representing a message overview. -# - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo) -# tuple. -# - the "internal" methods have been marked private (they now start with -# an underscore) - -# Other changes from the 2.x/3.1 nntplib: -# - automatic querying of capabilities at connect -# - New method NNTP.getcapabilities() -# - New method NNTP.over() -# - New helper function decode_header() -# - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and -# arbitrary iterables yielding lines. -# - An extensive test suite :-) - -# TODO: -# - return structured data (GroupInfo etc.) everywhere -# - support HDR - -# Imports -import re -import socket -import collections -import datetime -import sys -import warnings - -try: - import ssl -except ImportError: - _have_ssl = False -else: - _have_ssl = True - -from email.header import decode_header as _email_decode_header -from socket import _GLOBAL_DEFAULT_TIMEOUT - -__all__ = ["NNTP", - "NNTPError", "NNTPReplyError", "NNTPTemporaryError", - "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError", - "decode_header", - ] - -warnings._deprecated(__name__, remove=(3, 13)) - -# maximal line length when calling readline(). This is to prevent -# reading arbitrary length lines. RFC 3977 limits NNTP line length to -# 512 characters, including CRLF. We have selected 2048 just to be on -# the safe side. -_MAXLINE = 2048 - - -# Exceptions raised when an error or invalid response is received -class NNTPError(Exception): - """Base class for all nntplib exceptions""" - def __init__(self, *args): - Exception.__init__(self, *args) - try: - self.response = args[0] - except IndexError: - self.response = 'No response given' - -class NNTPReplyError(NNTPError): - """Unexpected [123]xx reply""" - pass - -class NNTPTemporaryError(NNTPError): - """4xx errors""" - pass - -class NNTPPermanentError(NNTPError): - """5xx errors""" - pass - -class NNTPProtocolError(NNTPError): - """Response does not begin with [1-5]""" - pass - -class NNTPDataError(NNTPError): - """Error in response data""" - pass - - -# Standard port used by NNTP servers -NNTP_PORT = 119 -NNTP_SSL_PORT = 563 - -# Response numbers that are followed by additional text (e.g. article) -_LONGRESP = { - '100', # HELP - '101', # CAPABILITIES - '211', # LISTGROUP (also not multi-line with GROUP) - '215', # LIST - '220', # ARTICLE - '221', # HEAD, XHDR - '222', # BODY - '224', # OVER, XOVER - '225', # HDR - '230', # NEWNEWS - '231', # NEWGROUPS - '282', # XGTITLE -} - -# Default decoded value for LIST OVERVIEW.FMT if not supported -_DEFAULT_OVERVIEW_FMT = [ - "subject", "from", "date", "message-id", "references", ":bytes", ":lines"] - -# Alternative names allowed in LIST OVERVIEW.FMT response -_OVERVIEW_FMT_ALTERNATIVES = { - 'bytes': ':bytes', - 'lines': ':lines', -} - -# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) -_CRLF = b'\r\n' - -GroupInfo = collections.namedtuple('GroupInfo', - ['group', 'last', 'first', 'flag']) - -ArticleInfo = collections.namedtuple('ArticleInfo', - ['number', 'message_id', 'lines']) - - -# Helper function(s) -def decode_header(header_str): - """Takes a unicode string representing a munged header value - and decodes it as a (possibly non-ASCII) readable value.""" - parts = [] - for v, enc in _email_decode_header(header_str): - if isinstance(v, bytes): - parts.append(v.decode(enc or 'ascii')) - else: - parts.append(v) - return ''.join(parts) - -def _parse_overview_fmt(lines): - """Parse a list of string representing the response to LIST OVERVIEW.FMT - and return a list of header/metadata names. - Raises NNTPDataError if the response is not compliant - (cf. RFC 3977, section 8.4).""" - fmt = [] - for line in lines: - if line[0] == ':': - # Metadata name (e.g. ":bytes") - name, _, suffix = line[1:].partition(':') - name = ':' + name - else: - # Header name (e.g. "Subject:" or "Xref:full") - name, _, suffix = line.partition(':') - name = name.lower() - name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name) - # Should we do something with the suffix? - fmt.append(name) - defaults = _DEFAULT_OVERVIEW_FMT - if len(fmt) < len(defaults): - raise NNTPDataError("LIST OVERVIEW.FMT response too short") - if fmt[:len(defaults)] != defaults: - raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields") - return fmt - -def _parse_overview(lines, fmt, data_process_func=None): - """Parse the response to an OVER or XOVER command according to the - overview format `fmt`.""" - n_defaults = len(_DEFAULT_OVERVIEW_FMT) - overview = [] - for line in lines: - fields = {} - article_number, *tokens = line.split('\t') - article_number = int(article_number) - for i, token in enumerate(tokens): - if i >= len(fmt): - # XXX should we raise an error? Some servers might not - # support LIST OVERVIEW.FMT and still return additional - # headers. - continue - field_name = fmt[i] - is_metadata = field_name.startswith(':') - if i >= n_defaults and not is_metadata: - # Non-default header names are included in full in the response - # (unless the field is totally empty) - h = field_name + ": " - if token and token[:len(h)].lower() != h: - raise NNTPDataError("OVER/XOVER response doesn't include " - "names of additional headers") - token = token[len(h):] if token else None - fields[fmt[i]] = token - overview.append((article_number, fields)) - return overview - -def _parse_datetime(date_str, time_str=None): - """Parse a pair of (date, time) strings, and return a datetime object. - If only the date is given, it is assumed to be date and time - concatenated together (e.g. response to the DATE command). - """ - if time_str is None: - time_str = date_str[-6:] - date_str = date_str[:-6] - hours = int(time_str[:2]) - minutes = int(time_str[2:4]) - seconds = int(time_str[4:]) - year = int(date_str[:-4]) - month = int(date_str[-4:-2]) - day = int(date_str[-2:]) - # RFC 3977 doesn't say how to interpret 2-char years. Assume that - # there are no dates before 1970 on Usenet. - if year < 70: - year += 2000 - elif year < 100: - year += 1900 - return datetime.datetime(year, month, day, hours, minutes, seconds) - -def _unparse_datetime(dt, legacy=False): - """Format a date or datetime object as a pair of (date, time) strings - in the format required by the NEWNEWS and NEWGROUPS commands. If a - date object is passed, the time is assumed to be midnight (00h00). - - The returned representation depends on the legacy flag: - * if legacy is False (the default): - date has the YYYYMMDD format and time the HHMMSS format - * if legacy is True: - date has the YYMMDD format and time the HHMMSS format. - RFC 3977 compliant servers should understand both formats; therefore, - legacy is only needed when talking to old servers. - """ - if not isinstance(dt, datetime.datetime): - time_str = "000000" - else: - time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt) - y = dt.year - if legacy: - y = y % 100 - date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt) - else: - date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt) - return date_str, time_str - - -if _have_ssl: - - def _encrypt_on(sock, context, hostname): - """Wrap a socket in SSL/TLS. Arguments: - - sock: Socket to wrap - - context: SSL context to use for the encrypted connection - Returns: - - sock: New, encrypted socket. - """ - # Generate a default SSL context if none was passed. - if context is None: - context = ssl._create_stdlib_context() - return context.wrap_socket(sock, server_hostname=hostname) - - -# The classes themselves -class NNTP: - # UTF-8 is the character set for all NNTP commands and responses: they - # are automatically encoded (when sending) and decoded (and receiving) - # by this class. - # However, some multi-line data blocks can contain arbitrary bytes (for - # example, latin-1 or utf-16 data in the body of a message). Commands - # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message - # data will therefore only accept and produce bytes objects. - # Furthermore, since there could be non-compliant servers out there, - # we use 'surrogateescape' as the error handler for fault tolerance - # and easy round-tripping. This could be useful for some applications - # (e.g. NNTP gateways). - - encoding = 'utf-8' - errors = 'surrogateescape' - - def __init__(self, host, port=NNTP_PORT, user=None, password=None, - readermode=None, usenetrc=False, - timeout=_GLOBAL_DEFAULT_TIMEOUT): - """Initialize an instance. Arguments: - - host: hostname to connect to - - port: port to connect to (default the standard NNTP port) - - user: username to authenticate with - - password: password to use with username - - readermode: if true, send 'mode reader' command after - connecting. - - usenetrc: allow loading username and password from ~/.netrc file - if not specified explicitly - - timeout: timeout (in seconds) used for socket connections - - readermode is sometimes necessary if you are connecting to an - NNTP server on the local machine and intend to call - reader-specific commands, such as `group'. If you get - unexpected NNTPPermanentErrors, you might need to set - readermode. - """ - self.host = host - self.port = port - self.sock = self._create_socket(timeout) - self.file = None - try: - self.file = self.sock.makefile("rwb") - self._base_init(readermode) - if user or usenetrc: - self.login(user, password, usenetrc) - except: - if self.file: - self.file.close() - self.sock.close() - raise - - def _base_init(self, readermode): - """Partial initialization for the NNTP protocol. - This instance method is extracted for supporting the test code. - """ - self.debugging = 0 - self.welcome = self._getresp() - - # Inquire about capabilities (RFC 3977). - self._caps = None - self.getcapabilities() - - # 'MODE READER' is sometimes necessary to enable 'reader' mode. - # However, the order in which 'MODE READER' and 'AUTHINFO' need to - # arrive differs between some NNTP servers. If _setreadermode() fails - # with an authorization failed error, it will set this to True; - # the login() routine will interpret that as a request to try again - # after performing its normal function. - # Enable only if we're not already in READER mode anyway. - self.readermode_afterauth = False - if readermode and 'READER' not in self._caps: - self._setreadermode() - if not self.readermode_afterauth: - # Capabilities might have changed after MODE READER - self._caps = None - self.getcapabilities() - - # RFC 4642 2.2.2: Both the client and the server MUST know if there is - # a TLS session active. A client MUST NOT attempt to start a TLS - # session if a TLS session is already active. - self.tls_on = False - - # Log in and encryption setup order is left to subclasses. - self.authenticated = False - - def __enter__(self): - return self - - def __exit__(self, *args): - is_connected = lambda: hasattr(self, "file") - if is_connected(): - try: - self.quit() - except (OSError, EOFError): - pass - finally: - if is_connected(): - self._close() - - def _create_socket(self, timeout): - if timeout is not None and not timeout: - raise ValueError('Non-blocking socket (timeout=0) is not supported') - sys.audit("nntplib.connect", self, self.host, self.port) - return socket.create_connection((self.host, self.port), timeout) - - def getwelcome(self): - """Get the welcome message from the server - (this is read and squirreled away by __init__()). - If the response code is 200, posting is allowed; - if it 201, posting is not allowed.""" - - if self.debugging: print('*welcome*', repr(self.welcome)) - return self.welcome - - def getcapabilities(self): - """Get the server capabilities, as read by __init__(). - If the CAPABILITIES command is not supported, an empty dict is - returned.""" - if self._caps is None: - self.nntp_version = 1 - self.nntp_implementation = None - try: - resp, caps = self.capabilities() - except (NNTPPermanentError, NNTPTemporaryError): - # Server doesn't support capabilities - self._caps = {} - else: - self._caps = caps - if 'VERSION' in caps: - # The server can advertise several supported versions, - # choose the highest. - self.nntp_version = max(map(int, caps['VERSION'])) - if 'IMPLEMENTATION' in caps: - self.nntp_implementation = ' '.join(caps['IMPLEMENTATION']) - return self._caps - - def set_debuglevel(self, level): - """Set the debugging level. Argument 'level' means: - 0: no debugging output (default) - 1: print commands and responses but not body text etc. - 2: also print raw lines read and sent before stripping CR/LF""" - - self.debugging = level - debug = set_debuglevel - - def _putline(self, line): - """Internal: send one line to the server, appending CRLF. - The `line` must be a bytes-like object.""" - sys.audit("nntplib.putline", self, line) - line = line + _CRLF - if self.debugging > 1: print('*put*', repr(line)) - self.file.write(line) - self.file.flush() - - def _putcmd(self, line): - """Internal: send one command to the server (through _putline()). - The `line` must be a unicode string.""" - if self.debugging: print('*cmd*', repr(line)) - line = line.encode(self.encoding, self.errors) - self._putline(line) - - def _getline(self, strip_crlf=True): - """Internal: return one line from the server, stripping _CRLF. - Raise EOFError if the connection is closed. - Returns a bytes object.""" - line = self.file.readline(_MAXLINE +1) - if len(line) > _MAXLINE: - raise NNTPDataError('line too long') - if self.debugging > 1: - print('*get*', repr(line)) - if not line: raise EOFError - if strip_crlf: - if line[-2:] == _CRLF: - line = line[:-2] - elif line[-1:] in _CRLF: - line = line[:-1] - return line - - def _getresp(self): - """Internal: get a response from the server. - Raise various errors if the response indicates an error. - Returns a unicode string.""" - resp = self._getline() - if self.debugging: print('*resp*', repr(resp)) - resp = resp.decode(self.encoding, self.errors) - c = resp[:1] - if c == '4': - raise NNTPTemporaryError(resp) - if c == '5': - raise NNTPPermanentError(resp) - if c not in '123': - raise NNTPProtocolError(resp) - return resp - - def _getlongresp(self, file=None): - """Internal: get a response plus following text from the server. - Raise various errors if the response indicates an error. - - Returns a (response, lines) tuple where `response` is a unicode - string and `lines` is a list of bytes objects. - If `file` is a file-like object, it must be open in binary mode. - """ - - openedFile = None - try: - # If a string was passed then open a file with that name - if isinstance(file, (str, bytes)): - openedFile = file = open(file, "wb") - - resp = self._getresp() - if resp[:3] not in _LONGRESP: - raise NNTPReplyError(resp) - - lines = [] - if file is not None: - # XXX lines = None instead? - terminators = (b'.' + _CRLF, b'.\n') - while 1: - line = self._getline(False) - if line in terminators: - break - if line.startswith(b'..'): - line = line[1:] - file.write(line) - else: - terminator = b'.' - while 1: - line = self._getline() - if line == terminator: - break - if line.startswith(b'..'): - line = line[1:] - lines.append(line) - finally: - # If this method created the file, then it must close it - if openedFile: - openedFile.close() - - return resp, lines - - def _shortcmd(self, line): - """Internal: send a command and get the response. - Same return value as _getresp().""" - self._putcmd(line) - return self._getresp() - - def _longcmd(self, line, file=None): - """Internal: send a command and get the response plus following text. - Same return value as _getlongresp().""" - self._putcmd(line) - return self._getlongresp(file) - - def _longcmdstring(self, line, file=None): - """Internal: send a command and get the response plus following text. - Same as _longcmd() and _getlongresp(), except that the returned `lines` - are unicode strings rather than bytes objects. - """ - self._putcmd(line) - resp, list = self._getlongresp(file) - return resp, [line.decode(self.encoding, self.errors) - for line in list] - - def _getoverviewfmt(self): - """Internal: get the overview format. Queries the server if not - already done, else returns the cached value.""" - try: - return self._cachedoverviewfmt - except AttributeError: - pass - try: - resp, lines = self._longcmdstring("LIST OVERVIEW.FMT") - except NNTPPermanentError: - # Not supported by server? - fmt = _DEFAULT_OVERVIEW_FMT[:] - else: - fmt = _parse_overview_fmt(lines) - self._cachedoverviewfmt = fmt - return fmt - - def _grouplist(self, lines): - # Parse lines into "group last first flag" - return [GroupInfo(*line.split()) for line in lines] - - def capabilities(self): - """Process a CAPABILITIES command. Not supported by all servers. - Return: - - resp: server response if successful - - caps: a dictionary mapping capability names to lists of tokens - (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] }) - """ - caps = {} - resp, lines = self._longcmdstring("CAPABILITIES") - for line in lines: - name, *tokens = line.split() - caps[name] = tokens - return resp, caps - - def newgroups(self, date, *, file=None): - """Process a NEWGROUPS command. Arguments: - - date: a date or datetime object - Return: - - resp: server response if successful - - list: list of newsgroup names - """ - if not isinstance(date, (datetime.date, datetime.date)): - raise TypeError( - "the date parameter must be a date or datetime object, " - "not '{:40}'".format(date.__class__.__name__)) - date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) - cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str) - resp, lines = self._longcmdstring(cmd, file) - return resp, self._grouplist(lines) - - def newnews(self, group, date, *, file=None): - """Process a NEWNEWS command. Arguments: - - group: group name or '*' - - date: a date or datetime object - Return: - - resp: server response if successful - - list: list of message ids - """ - if not isinstance(date, (datetime.date, datetime.date)): - raise TypeError( - "the date parameter must be a date or datetime object, " - "not '{:40}'".format(date.__class__.__name__)) - date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) - cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str) - return self._longcmdstring(cmd, file) - - def list(self, group_pattern=None, *, file=None): - """Process a LIST or LIST ACTIVE command. Arguments: - - group_pattern: a pattern indicating which groups to query - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of (group, last, first, flag) (strings) - """ - if group_pattern is not None: - command = 'LIST ACTIVE ' + group_pattern - else: - command = 'LIST' - resp, lines = self._longcmdstring(command, file) - return resp, self._grouplist(lines) - - def _getdescriptions(self, group_pattern, return_all): - line_pat = re.compile('^(?P[^ \t]+)[ \t]+(.*)$') - # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first - resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern) - if not resp.startswith('215'): - # Now the deprecated XGTITLE. This either raises an error - # or succeeds with the same output structure as LIST - # NEWSGROUPS. - resp, lines = self._longcmdstring('XGTITLE ' + group_pattern) - groups = {} - for raw_line in lines: - match = line_pat.search(raw_line.strip()) - if match: - name, desc = match.group(1, 2) - if not return_all: - return desc - groups[name] = desc - if return_all: - return resp, groups - else: - # Nothing found - return '' - - def description(self, group): - """Get a description for a single group. If more than one - group matches ('group' is a pattern), return the first. If no - group matches, return an empty string. - - This elides the response code from the server, since it can - only be '215' or '285' (for xgtitle) anyway. If the response - code is needed, use the 'descriptions' method. - - NOTE: This neither checks for a wildcard in 'group' nor does - it check whether the group actually exists.""" - return self._getdescriptions(group, False) - - def descriptions(self, group_pattern): - """Get descriptions for a range of groups.""" - return self._getdescriptions(group_pattern, True) - - def group(self, name): - """Process a GROUP command. Argument: - - group: the group name - Returns: - - resp: server response if successful - - count: number of articles - - first: first article number - - last: last article number - - name: the group name - """ - resp = self._shortcmd('GROUP ' + name) - if not resp.startswith('211'): - raise NNTPReplyError(resp) - words = resp.split() - count = first = last = 0 - n = len(words) - if n > 1: - count = words[1] - if n > 2: - first = words[2] - if n > 3: - last = words[3] - if n > 4: - name = words[4].lower() - return resp, int(count), int(first), int(last), name - - def help(self, *, file=None): - """Process a HELP command. Argument: - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of strings returned by the server in response to the - HELP command - """ - return self._longcmdstring('HELP', file) - - def _statparse(self, resp): - """Internal: parse the response line of a STAT, NEXT, LAST, - ARTICLE, HEAD or BODY command.""" - if not resp.startswith('22'): - raise NNTPReplyError(resp) - words = resp.split() - art_num = int(words[1]) - message_id = words[2] - return resp, art_num, message_id - - def _statcmd(self, line): - """Internal: process a STAT, NEXT or LAST command.""" - resp = self._shortcmd(line) - return self._statparse(resp) - - def stat(self, message_spec=None): - """Process a STAT command. Argument: - - message_spec: article number or message id (if not specified, - the current article is selected) - Returns: - - resp: server response if successful - - art_num: the article number - - message_id: the message id - """ - if message_spec: - return self._statcmd('STAT {0}'.format(message_spec)) - else: - return self._statcmd('STAT') - - def next(self): - """Process a NEXT command. No arguments. Return as for STAT.""" - return self._statcmd('NEXT') - - def last(self): - """Process a LAST command. No arguments. Return as for STAT.""" - return self._statcmd('LAST') - - def _artcmd(self, line, file=None): - """Internal: process a HEAD, BODY or ARTICLE command.""" - resp, lines = self._longcmd(line, file) - resp, art_num, message_id = self._statparse(resp) - return resp, ArticleInfo(art_num, message_id, lines) - - def head(self, message_spec=None, *, file=None): - """Process a HEAD command. Argument: - - message_spec: article number or message id - - file: filename string or file object to store the headers in - Returns: - - resp: server response if successful - - ArticleInfo: (article number, message id, list of header lines) - """ - if message_spec is not None: - cmd = 'HEAD {0}'.format(message_spec) - else: - cmd = 'HEAD' - return self._artcmd(cmd, file) - - def body(self, message_spec=None, *, file=None): - """Process a BODY command. Argument: - - message_spec: article number or message id - - file: filename string or file object to store the body in - Returns: - - resp: server response if successful - - ArticleInfo: (article number, message id, list of body lines) - """ - if message_spec is not None: - cmd = 'BODY {0}'.format(message_spec) - else: - cmd = 'BODY' - return self._artcmd(cmd, file) - - def article(self, message_spec=None, *, file=None): - """Process an ARTICLE command. Argument: - - message_spec: article number or message id - - file: filename string or file object to store the article in - Returns: - - resp: server response if successful - - ArticleInfo: (article number, message id, list of article lines) - """ - if message_spec is not None: - cmd = 'ARTICLE {0}'.format(message_spec) - else: - cmd = 'ARTICLE' - return self._artcmd(cmd, file) - - def slave(self): - """Process a SLAVE command. Returns: - - resp: server response if successful - """ - return self._shortcmd('SLAVE') - - def xhdr(self, hdr, str, *, file=None): - """Process an XHDR command (optional server extension). Arguments: - - hdr: the header type (e.g. 'subject') - - str: an article nr, a message id, or a range nr1-nr2 - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of (nr, value) strings - """ - pat = re.compile('^([0-9]+) ?(.*)\n?') - resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file) - def remove_number(line): - m = pat.match(line) - return m.group(1, 2) if m else line - return resp, [remove_number(line) for line in lines] - - def xover(self, start, end, *, file=None): - """Process an XOVER command (optional server extension) Arguments: - - start: start of range - - end: end of range - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of dicts containing the response fields - """ - resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end), - file) - fmt = self._getoverviewfmt() - return resp, _parse_overview(lines, fmt) - - def over(self, message_spec, *, file=None): - """Process an OVER command. If the command isn't supported, fall - back to XOVER. Arguments: - - message_spec: - - either a message id, indicating the article to fetch - information about - - or a (start, end) tuple, indicating a range of article numbers; - if end is None, information up to the newest message will be - retrieved - - or None, indicating the current article number must be used - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of dicts containing the response fields - - NOTE: the "message id" form isn't supported by XOVER - """ - cmd = 'OVER' if 'OVER' in self._caps else 'XOVER' - if isinstance(message_spec, (tuple, list)): - start, end = message_spec - cmd += ' {0}-{1}'.format(start, end or '') - elif message_spec is not None: - cmd = cmd + ' ' + message_spec - resp, lines = self._longcmdstring(cmd, file) - fmt = self._getoverviewfmt() - return resp, _parse_overview(lines, fmt) - - def date(self): - """Process the DATE command. - Returns: - - resp: server response if successful - - date: datetime object - """ - resp = self._shortcmd("DATE") - if not resp.startswith('111'): - raise NNTPReplyError(resp) - elem = resp.split() - if len(elem) != 2: - raise NNTPDataError(resp) - date = elem[1] - if len(date) != 14: - raise NNTPDataError(resp) - return resp, _parse_datetime(date, None) - - def _post(self, command, f): - resp = self._shortcmd(command) - # Raises a specific exception if posting is not allowed - if not resp.startswith('3'): - raise NNTPReplyError(resp) - if isinstance(f, (bytes, bytearray)): - f = f.splitlines() - # We don't use _putline() because: - # - we don't want additional CRLF if the file or iterable is already - # in the right format - # - we don't want a spurious flush() after each line is written - for line in f: - if not line.endswith(_CRLF): - line = line.rstrip(b"\r\n") + _CRLF - if line.startswith(b'.'): - line = b'.' + line - self.file.write(line) - self.file.write(b".\r\n") - self.file.flush() - return self._getresp() - - def post(self, data): - """Process a POST command. Arguments: - - data: bytes object, iterable or file containing the article - Returns: - - resp: server response if successful""" - return self._post('POST', data) - - def ihave(self, message_id, data): - """Process an IHAVE command. Arguments: - - message_id: message-id of the article - - data: file containing the article - Returns: - - resp: server response if successful - Note that if the server refuses the article an exception is raised.""" - return self._post('IHAVE {0}'.format(message_id), data) - - def _close(self): - try: - if self.file: - self.file.close() - del self.file - finally: - self.sock.close() - - def quit(self): - """Process a QUIT command and close the socket. Returns: - - resp: server response if successful""" - try: - resp = self._shortcmd('QUIT') - finally: - self._close() - return resp - - def login(self, user=None, password=None, usenetrc=True): - if self.authenticated: - raise ValueError("Already logged in.") - if not user and not usenetrc: - raise ValueError( - "At least one of `user` and `usenetrc` must be specified") - # If no login/password was specified but netrc was requested, - # try to get them from ~/.netrc - # Presume that if .netrc has an entry, NNRP authentication is required. - try: - if usenetrc and not user: - import netrc - credentials = netrc.netrc() - auth = credentials.authenticators(self.host) - if auth: - user = auth[0] - password = auth[2] - except OSError: - pass - # Perform NNTP authentication if needed. - if not user: - return - resp = self._shortcmd('authinfo user ' + user) - if resp.startswith('381'): - if not password: - raise NNTPReplyError(resp) - else: - resp = self._shortcmd('authinfo pass ' + password) - if not resp.startswith('281'): - raise NNTPPermanentError(resp) - # Capabilities might have changed after login - self._caps = None - self.getcapabilities() - # Attempt to send mode reader if it was requested after login. - # Only do so if we're not in reader mode already. - if self.readermode_afterauth and 'READER' not in self._caps: - self._setreadermode() - # Capabilities might have changed after MODE READER - self._caps = None - self.getcapabilities() - - def _setreadermode(self): - try: - self.welcome = self._shortcmd('mode reader') - except NNTPPermanentError: - # Error 5xx, probably 'not implemented' - pass - except NNTPTemporaryError as e: - if e.response.startswith('480'): - # Need authorization before 'mode reader' - self.readermode_afterauth = True - else: - raise - - if _have_ssl: - def starttls(self, context=None): - """Process a STARTTLS command. Arguments: - - context: SSL context to use for the encrypted connection - """ - # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if - # a TLS session already exists. - if self.tls_on: - raise ValueError("TLS is already enabled.") - if self.authenticated: - raise ValueError("TLS cannot be started after authentication.") - resp = self._shortcmd('STARTTLS') - if resp.startswith('382'): - self.file.close() - self.sock = _encrypt_on(self.sock, context, self.host) - self.file = self.sock.makefile("rwb") - self.tls_on = True - # Capabilities may change after TLS starts up, so ask for them - # again. - self._caps = None - self.getcapabilities() - else: - raise NNTPError("TLS failed to start.") - - -if _have_ssl: - class NNTP_SSL(NNTP): - - def __init__(self, host, port=NNTP_SSL_PORT, - user=None, password=None, ssl_context=None, - readermode=None, usenetrc=False, - timeout=_GLOBAL_DEFAULT_TIMEOUT): - """This works identically to NNTP.__init__, except for the change - in default port and the `ssl_context` argument for SSL connections. - """ - self.ssl_context = ssl_context - super().__init__(host, port, user, password, readermode, - usenetrc, timeout) - - def _create_socket(self, timeout): - sock = super()._create_socket(timeout) - try: - sock = _encrypt_on(sock, self.ssl_context, self.host) - except: - sock.close() - raise - else: - return sock - - __all__.append("NNTP_SSL") - - -# Test retrieval when run as a script. -if __name__ == '__main__': - import argparse - - parser = argparse.ArgumentParser(description="""\ - nntplib built-in demo - display the latest articles in a newsgroup""") - parser.add_argument('-g', '--group', default='gmane.comp.python.general', - help='group to fetch messages from (default: %(default)s)') - parser.add_argument('-s', '--server', default='news.gmane.io', - help='NNTP server hostname (default: %(default)s)') - parser.add_argument('-p', '--port', default=-1, type=int, - help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT)) - parser.add_argument('-n', '--nb-articles', default=10, type=int, - help='number of articles to fetch (default: %(default)s)') - parser.add_argument('-S', '--ssl', action='store_true', default=False, - help='use NNTP over SSL') - args = parser.parse_args() - - port = args.port - if not args.ssl: - if port == -1: - port = NNTP_PORT - s = NNTP(host=args.server, port=port) - else: - if port == -1: - port = NNTP_SSL_PORT - s = NNTP_SSL(host=args.server, port=port) - - caps = s.getcapabilities() - if 'STARTTLS' in caps: - s.starttls() - resp, count, first, last, name = s.group(args.group) - print('Group', name, 'has', count, 'articles, range', first, 'to', last) - - def cut(s, lim): - if len(s) > lim: - s = s[:lim - 4] + "..." - return s - - first = str(int(last) - args.nb_articles + 1) - resp, overviews = s.xover(first, last) - for artnum, over in overviews: - author = decode_header(over['from']).split('<', 1)[0] - subject = decode_header(over['subject']) - lines = int(over[':lines']) - print("{:7} {:20} {:42} ({})".format( - artnum, cut(author, 20), cut(subject, 42), lines) - ) - - s.quit() diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 0f3674fe11eecd..3061a4a5ef4c56 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -23,7 +23,6 @@ import genericpath from genericpath import * - __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", @@ -601,7 +600,7 @@ def abspath(path): return _abspath_fallback(path) try: - from nt import _getfinalpathname, readlink as _nt_readlink + from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink except ImportError: # realpath is a no-op on systems without _getfinalpathname support. realpath = abspath @@ -688,10 +687,15 @@ def _getfinalpathname_nonstrict(path): except OSError: # If we fail to readlink(), let's keep traversing pass - path, name = split(path) - # TODO (bpo-38186): Request the real file name from the directory - # entry using FindFirstFileW. For now, we will return the path - # as best we have it + # If we get these errors, try to get the real name of the file without accessing it. + if ex.winerror in (1, 5, 32, 50, 87, 1920, 1921): + try: + name = _findfirstfile(path) + path, _ = split(path) + except OSError: + path, name = split(path) + else: + path, name = split(path) if path and not name: return path + tail tail = join(name, tail) if tail else name @@ -721,6 +725,14 @@ def realpath(path, *, strict=False): try: path = _getfinalpathname(path) initial_winerror = 0 + except ValueError as ex: + # gh-106242: Raised for embedded null characters + # In strict mode, we convert into an OSError. + # Non-strict mode returns the path as-is, since we've already + # made it absolute. + if strict: + raise OSError(str(ex)) from None + path = normpath(path) except OSError as ex: if strict: raise @@ -740,6 +752,10 @@ def realpath(path, *, strict=False): try: if _getfinalpathname(spath) == path: path = spath + except ValueError as ex: + # Unexpected, as an invalid path should not have gained a prefix + # at any point, but we ignore this error just in case. + pass except OSError as ex: # If the path does not exist and originally did not exist, then # strip the prefix anyway. @@ -867,3 +883,19 @@ def commonpath(paths): except ImportError: # Use genericpath.* as imported above pass + + +try: + from nt import _path_isdevdrive +except ImportError: + def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive.""" + # Never a Dev Drive + return False +else: + def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive.""" + try: + return _path_isdevdrive(abspath(path)) + except OSError: + return False diff --git a/Lib/opcode.py b/Lib/opcode.py index 155466b1ab85f1..88f4df7c0e8c38 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,428 +4,41 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG"] -# It's a chicken-and-egg I'm afraid: -# We're imported before _opcode's made. -# With exception unheeded -# (stack_effect is not needed) -# Both our chickens and eggs are allayed. -# --Larry Hastings, 2013/11/23 +__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap", + "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname", + "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"] -try: - from _opcode import stack_effect - __all__.append('stack_effect') -except ImportError: - pass +import _opcode +from _opcode import stack_effect -cmp_op = ('<', '<=', '==', '!=', '>', '>=') - -hasarg = [] -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasexc = [] - - -ENABLE_SPECIALIZATION = True - -def is_pseudo(op): - return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE - -oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, - haslocal, hascompare, hasfree, hasexc] - -opmap = {} - -## pseudo opcodes (used in the compiler) mapped to the values -## they can become in the actual code. -_pseudo_ops = {} - -def def_op(name, op): - opmap[name] = op - -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) - -def pseudo_op(name, op, real_ops): - def_op(name, op) - _pseudo_ops[name] = real_ops - # add the pseudo opcode to the lists its targets are in - for oplist in oplists: - res = [opmap[rop] in oplist for rop in real_ops] - if any(res): - assert all(res) - oplist.append(op) - - -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('CACHE', 0) -def_op('POP_TOP', 1) -def_op('PUSH_NULL', 2) -def_op('INTERPRETER_EXIT', 3) - -def_op('END_FOR', 4) -def_op('END_SEND', 5) - -def_op('NOP', 9) - -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) - -# We reserve 17 as it is the initial value for the specializing counter -# This helps us catch cases where we attempt to execute a cache. -def_op('RESERVED', 17) - -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_SLICE', 26) -def_op('STORE_SLICE', 27) - -def_op('GET_LEN', 30) -def_op('MATCH_MAPPING', 31) -def_op('MATCH_SEQUENCE', 32) -def_op('MATCH_KEYS', 33) - -def_op('PUSH_EXC_INFO', 35) -def_op('CHECK_EXC_MATCH', 36) -def_op('CHECK_EG_MATCH', 37) - -def_op('WITH_EXCEPT_START', 49) -def_op('GET_AITER', 50) -def_op('GET_ANEXT', 51) -def_op('BEFORE_ASYNC_WITH', 52) -def_op('BEFORE_WITH', 53) -def_op('END_ASYNC_FOR', 54) -def_op('CLEANUP_THROW', 55) - -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) - -def_op('GET_ITER', 68) -def_op('GET_YIELD_FROM_ITER', 69) - -def_op('LOAD_BUILD_CLASS', 71) - -def_op('LOAD_ASSERTION_ERROR', 74) -def_op('RETURN_GENERATOR', 75) - -def_op('RETURN_VALUE', 83) - -def_op('SETUP_ANNOTATIONS', 85) - -def_op('POP_EXCEPT', 89) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +EXTENDED_ARG = opmap['EXTENDED_ARG'] -HAVE_ARGUMENT = 90 # real opcodes from here have an argument: - -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" -def_op('SWAP', 99) -def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries -name_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list -jrel_op('JUMP_FORWARD', 110) # Number of words to skip -jrel_op('POP_JUMP_IF_FALSE', 114) -jrel_op('POP_JUMP_IF_TRUE', 115) -name_op('LOAD_GLOBAL', 116) # Index in name list -def_op('IS_OP', 117) -def_op('CONTAINS_OP', 118) -def_op('RERAISE', 119) -def_op('COPY', 120) -def_op('RETURN_CONST', 121) -hasconst.append(121) -def_op('BINARY_OP', 122) -jrel_op('SEND', 123) # Number of words to skip -def_op('LOAD_FAST', 124) # Local variable number, no null check -haslocal.append(124) -def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) -def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) -def_op('LOAD_FAST_CHECK', 127) # Local variable number -haslocal.append(127) -jrel_op('POP_JUMP_IF_NOT_NONE', 128) -jrel_op('POP_JUMP_IF_NONE', 129) -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('GET_AWAITABLE', 131) -def_op('MAKE_FUNCTION', 132) # Flags -def_op('BUILD_SLICE', 133) # Number of items -jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) -def_op('MAKE_CELL', 135) -hasfree.append(135) -def_op('LOAD_CLOSURE', 136) -hasfree.append(136) -def_op('LOAD_DEREF', 137) -hasfree.append(137) -def_op('STORE_DEREF', 138) -hasfree.append(138) -def_op('DELETE_DEREF', 139) -hasfree.append(139) -jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -name_op('LOAD_SUPER_ATTR', 141) -def_op('CALL_FUNCTION_EX', 142) # Flags -def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number -haslocal.append(143) - -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) -def_op('LOAD_CLASSDEREF', 148) -hasfree.append(148) -def_op('COPY_FREE_VARS', 149) -def_op('YIELD_VALUE', 150) -def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py -def_op('MATCH_CLASS', 152) - -def_op('FORMAT_VALUE', 155) -def_op('BUILD_CONST_KEY_MAP', 156) -def_op('BUILD_STRING', 157) - -def_op('LIST_EXTEND', 162) -def_op('SET_UPDATE', 163) -def_op('DICT_MERGE', 164) -def_op('DICT_UPDATE', 165) - -def_op('CALL', 171) -def_op('KW_NAMES', 172) -hasconst.append(172) -def_op('CALL_INTRINSIC_1', 173) -def_op('CALL_INTRINSIC_2', 174) - -# Instrumented instructions -MIN_INSTRUMENTED_OPCODE = 238 - -def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 238) -def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 239) -def_op('INSTRUMENTED_RESUME', 240) -def_op('INSTRUMENTED_CALL', 241) -def_op('INSTRUMENTED_RETURN_VALUE', 242) -def_op('INSTRUMENTED_YIELD_VALUE', 243) -def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) -def_op('INSTRUMENTED_JUMP_FORWARD', 245) -def_op('INSTRUMENTED_JUMP_BACKWARD', 246) -def_op('INSTRUMENTED_RETURN_CONST', 247) -def_op('INSTRUMENTED_FOR_ITER', 248) -def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) -def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) -def_op('INSTRUMENTED_END_FOR', 251) -def_op('INSTRUMENTED_END_SEND', 252) -def_op('INSTRUMENTED_INSTRUCTION', 253) -def_op('INSTRUMENTED_LINE', 254) -# 255 is reserved - -hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) - -MIN_PSEUDO_OPCODE = 256 - -pseudo_op('SETUP_FINALLY', 256, ['NOP']) -hasexc.append(256) -pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -hasexc.append(257) -pseudo_op('SETUP_WITH', 258, ['NOP']) -hasexc.append(258) -pseudo_op('POP_BLOCK', 259, ['NOP']) - -pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) -pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) - -pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR']) -pseudo_op('LOAD_SUPER_METHOD', 263, ['LOAD_SUPER_ATTR']) -pseudo_op('LOAD_ZERO_SUPER_METHOD', 264, ['LOAD_SUPER_ATTR']) -pseudo_op('LOAD_ZERO_SUPER_ATTR', 265, ['LOAD_SUPER_ATTR']) - -pseudo_op('STORE_FAST_MAYBE_NULL', 266, ['STORE_FAST']) - -MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 - -del def_op, name_op, jrel_op, jabs_op, pseudo_op - -opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] for op, i in opmap.items(): opname[i] = op +cmp_op = ('<', '<=', '==', '!=', '>', '>=') -_nb_ops = [ - ("NB_ADD", "+"), - ("NB_AND", "&"), - ("NB_FLOOR_DIVIDE", "//"), - ("NB_LSHIFT", "<<"), - ("NB_MATRIX_MULTIPLY", "@"), - ("NB_MULTIPLY", "*"), - ("NB_REMAINDER", "%"), - ("NB_OR", "|"), - ("NB_POWER", "**"), - ("NB_RSHIFT", ">>"), - ("NB_SUBTRACT", "-"), - ("NB_TRUE_DIVIDE", "/"), - ("NB_XOR", "^"), - ("NB_INPLACE_ADD", "+="), - ("NB_INPLACE_AND", "&="), - ("NB_INPLACE_FLOOR_DIVIDE", "//="), - ("NB_INPLACE_LSHIFT", "<<="), - ("NB_INPLACE_MATRIX_MULTIPLY", "@="), - ("NB_INPLACE_MULTIPLY", "*="), - ("NB_INPLACE_REMAINDER", "%="), - ("NB_INPLACE_OR", "|="), - ("NB_INPLACE_POWER", "**="), - ("NB_INPLACE_RSHIFT", ">>="), - ("NB_INPLACE_SUBTRACT", "-="), - ("NB_INPLACE_TRUE_DIVIDE", "/="), - ("NB_INPLACE_XOR", "^="), -] +# These lists are documented as part of the dis module's API +hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] +hasconst = [op for op in opmap.values() if _opcode.has_const(op)] +hasname = [op for op in opmap.values() if _opcode.has_name(op)] +hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] +hasjrel = hasjump # for backward compatibility +hasjabs = [] +hasfree = [op for op in opmap.values() if _opcode.has_free(op)] +haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] -_intrinsic_1_descs = [ - "INTRINSIC_1_INVALID", - "INTRINSIC_PRINT", - "INTRINSIC_IMPORT_STAR", - "INTRINSIC_STOPITERATION_ERROR", - "INTRINSIC_ASYNC_GEN_WRAP", - "INTRINSIC_UNARY_POSITIVE", - "INTRINSIC_LIST_TO_TUPLE", -] -_intrinsic_2_descs = [ - 'INTRINSIC_2_INVALID', - 'INTRINSIC_PREP_RERAISE_STAR', - ] +_intrinsic_1_descs = _opcode.get_intrinsic1_descs() +_intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_nb_ops = _opcode.get_nb_ops() -_specializations = { - "BINARY_OP": [ - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_ADD_INT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_SUBTRACT_INT", - ], - "BINARY_SUBSCR": [ - "BINARY_SUBSCR_DICT", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_TUPLE_INT", - ], - "CALL": [ - "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_NO_KW_BUILTIN_FAST", - "CALL_NO_KW_BUILTIN_O", - "CALL_NO_KW_ISINSTANCE", - "CALL_NO_KW_LEN", - "CALL_NO_KW_LIST_APPEND", - "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "CALL_NO_KW_METHOD_DESCRIPTOR_O", - "CALL_NO_KW_STR_1", - "CALL_NO_KW_TUPLE_1", - "CALL_NO_KW_TYPE_1", - ], - "COMPARE_OP": [ - "COMPARE_OP_FLOAT", - "COMPARE_OP_INT", - "COMPARE_OP_STR", - ], - "FOR_ITER": [ - "FOR_ITER_LIST", - "FOR_ITER_TUPLE", - "FOR_ITER_RANGE", - "FOR_ITER_GEN", - ], - "LOAD_SUPER_ATTR": [ - "LOAD_SUPER_ATTR_ATTR", - "LOAD_SUPER_ATTR_METHOD", - ], - "LOAD_ATTR": [ - # These potentially push [NULL, bound method] onto the stack. - "LOAD_ATTR_CLASS", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_WITH_HINT", - # These will always push [unbound method, self] onto the stack. - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_WITH_VALUES", - ], - "LOAD_CONST": [ - "LOAD_CONST__LOAD_FAST", - ], - "LOAD_FAST": [ - "LOAD_FAST__LOAD_CONST", - "LOAD_FAST__LOAD_FAST", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_BUILTIN", - "LOAD_GLOBAL_MODULE", - ], - "STORE_ATTR": [ - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "STORE_FAST": [ - "STORE_FAST__LOAD_FAST", - "STORE_FAST__STORE_FAST", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_LIST", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_TWO_TUPLE", - ], - "SEND": [ - "SEND_GEN", - ], -} -_specialized_instructions = [ - opcode for family in _specializations.values() for opcode in family -] +hascompare = [opmap["COMPARE_OP"]] _cache_format = { "LOAD_GLOBAL": { @@ -473,8 +86,27 @@ def pseudo_op(name, op, real_ops): "SEND": { "counter": 1, }, + "JUMP_BACKWARD": { + "counter": 1, + }, + "TO_BOOL": { + "counter": 1, + "version": 2, + }, + "POP_JUMP_IF_TRUE": { + "counter": 1, + }, + "POP_JUMP_IF_FALSE": { + "counter": 1, + }, + "POP_JUMP_IF_NONE": { + "counter": 1, + }, + "POP_JUMP_IF_NOT_NONE": { + "counter": 1, + }, } -_inline_cache_entries = [ - sum(_cache_format.get(opname[opcode], {}).values()) for opcode in range(256) -] +_inline_cache_entries = { + name : sum(value.values()) for (name, value) in _cache_format.items() +} diff --git a/Lib/os.py b/Lib/os.py index 598c9e502301f7..a17946750ea7e7 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1061,6 +1061,12 @@ def _fspath(path): else: raise TypeError("expected str, bytes or os.PathLike object, " "not " + path_type.__name__) + except TypeError: + if path_type.__fspath__ is None: + raise TypeError("expected str, bytes or os.PathLike object, " + "not " + path_type.__name__) from None + else: + raise if isinstance(path_repr, (str, bytes)): return path_repr else: @@ -1079,6 +1085,8 @@ class PathLike(abc.ABC): """Abstract base class for implementing the file system path protocol.""" + __slots__ = () + @abc.abstractmethod def __fspath__(self): """Return the file system path representation of the object.""" @@ -1128,3 +1136,17 @@ def add_dll_directory(path): cookie, nt._remove_dll_directory ) + + +if _exists('sched_getaffinity') and sys._get_cpu_count_config() < 0: + def process_cpu_count(): + """ + Get the number of CPUs of the current process. + + Return the number of logical CPUs usable by the calling thread of the + current process. Return None if indeterminable. + """ + return len(sched_getaffinity(0)) +else: + # Just an alias to cpu_count() (same docstring) + process_cpu_count = cpu_count diff --git a/Lib/pathlib.py b/Lib/pathlib.py deleted file mode 100644 index 40b72930e1f08a..00000000000000 --- a/Lib/pathlib.py +++ /dev/null @@ -1,1356 +0,0 @@ -"""Object-oriented filesystem paths. - -This module provides classes to represent abstract paths and concrete -paths with operations that have semantics appropriate for different -operating systems. -""" - -import fnmatch -import functools -import io -import ntpath -import os -import posixpath -import re -import sys -import warnings -from _collections_abc import Sequence -from errno import ENOENT, ENOTDIR, EBADF, ELOOP -from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -from urllib.parse import quote_from_bytes as urlquote_from_bytes - - -__all__ = [ - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - -# -# Internals -# - -# Reference for Windows paths can be found at -# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . -_WIN_RESERVED_NAMES = frozenset( - {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | - {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | - {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} -) - -_WINERROR_NOT_READY = 21 # drive exists but is not accessible -_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 -_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself - -# EBADF - guard against macOS `stat` throwing EBADF -_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) - -_IGNORED_WINERRORS = ( - _WINERROR_NOT_READY, - _WINERROR_INVALID_NAME, - _WINERROR_CANT_RESOLVE_FILENAME) - -def _ignore_error(exception): - return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or - getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) - - -def _is_case_sensitive(flavour): - return flavour.normcase('Aa') == 'Aa' - -# -# Globbing helpers -# - -@functools.lru_cache() -def _make_selector(pattern_parts, flavour, case_sensitive): - pat = pattern_parts[0] - if not pat: - return _TerminatingSelector() - if pat == '**': - child_parts_idx = 1 - while child_parts_idx < len(pattern_parts) and pattern_parts[child_parts_idx] == '**': - child_parts_idx += 1 - child_parts = pattern_parts[child_parts_idx:] - if '**' in child_parts: - cls = _DoubleRecursiveWildcardSelector - else: - cls = _RecursiveWildcardSelector - else: - child_parts = pattern_parts[1:] - if pat == '..': - cls = _ParentSelector - elif '**' in pat: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - else: - cls = _WildcardSelector - return cls(pat, child_parts, flavour, case_sensitive) - - -class _Selector: - """A selector matches a specific glob pattern part against the children - of a given path.""" - - def __init__(self, child_parts, flavour, case_sensitive): - self.child_parts = child_parts - if child_parts: - self.successor = _make_selector(child_parts, flavour, case_sensitive) - self.dironly = True - else: - self.successor = _TerminatingSelector() - self.dironly = False - - def select_from(self, parent_path): - """Iterate over all child paths of `parent_path` matched by this - selector. This can contain parent_path itself.""" - path_cls = type(parent_path) - scandir = path_cls._scandir - if not parent_path.is_dir(): - return iter([]) - return self._select_from(parent_path, scandir) - - -class _TerminatingSelector: - - def _select_from(self, parent_path, scandir): - yield parent_path - - -class _ParentSelector(_Selector): - - def __init__(self, name, child_parts, flavour, case_sensitive): - _Selector.__init__(self, child_parts, flavour, case_sensitive) - - def _select_from(self, parent_path, scandir): - path = parent_path._make_child_relpath('..') - for p in self.successor._select_from(path, scandir): - yield p - - -class _WildcardSelector(_Selector): - - def __init__(self, pat, child_parts, flavour, case_sensitive): - _Selector.__init__(self, child_parts, flavour, case_sensitive) - if case_sensitive is None: - # TODO: evaluate case-sensitivity of each directory in _select_from() - case_sensitive = _is_case_sensitive(flavour) - flags = re.NOFLAG if case_sensitive else re.IGNORECASE - self.match = re.compile(fnmatch.translate(pat), flags=flags).fullmatch - - def _select_from(self, parent_path, scandir): - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with scandir(parent_path) as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - if self.dironly: - try: - if not entry.is_dir(): - continue - except OSError: - continue - name = entry.name - if self.match(name): - path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, scandir): - yield p - - -class _RecursiveWildcardSelector(_Selector): - - def __init__(self, pat, child_parts, flavour, case_sensitive): - _Selector.__init__(self, child_parts, flavour, case_sensitive) - - def _iterate_directories(self, parent_path, scandir): - yield parent_path - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with scandir(parent_path) as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - entry_is_dir = False - try: - entry_is_dir = entry.is_dir(follow_symlinks=False) - except OSError: - pass - if entry_is_dir: - path = parent_path._make_child_relpath(entry.name) - for p in self._iterate_directories(path, scandir): - yield p - - def _select_from(self, parent_path, scandir): - successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, scandir): - for p in successor_select(starting_point, scandir): - yield p - - -class _DoubleRecursiveWildcardSelector(_RecursiveWildcardSelector): - """ - Like _RecursiveWildcardSelector, but also de-duplicates results from - successive selectors. This is necessary if the pattern contains - multiple non-adjacent '**' segments. - """ - - def _select_from(self, parent_path, scandir): - yielded = set() - try: - for p in super()._select_from(parent_path, scandir): - if p not in yielded: - yield p - yielded.add(p) - finally: - yielded.clear() - - -# -# Public API -# - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') - - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail - - def __len__(self): - return len(self._tail) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) - - -class PurePath(object): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_raw_path` slot stores an unnormalized string path. This is set - # in the `__init__()` method. - '_raw_path', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', - - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - ) - _flavour = os.path - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __reduce__(self): - # Using the parts tuple helps share interned path parts - # when pickling related paths. - return (self.__class__, self.parts) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - path = arg._raw_path - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - if len(paths) == 0: - self._raw_path = '' - elif len(paths) == 1: - self._raw_path = paths[0] - else: - self._raw_path = self._flavour.join(*paths) - - def with_segments(self, *pathsegments): - """Construct a new path object from any number of path-like objects. - Subclasses may override this method to customize how new path objects - are created from methods like `iterdir()`. - """ - return type(self)(*pathsegments) - - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls._flavour.sep - altsep = cls._flavour.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls._flavour.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - def _load_parts(self): - drv, root, tail = self._parse_path(self._raw_path) - self._drv = drv - self._root = root - self._tail_cached = tail - - def _from_parsed_parts(self, drv, root, tail): - path_str = self._format_parsed_parts(drv, root, tail) - path = self.with_segments(path_str) - path._str = path_str or '.' - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls._flavour.sep.join(tail) - elif tail and cls._flavour.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls._flavour.sep.join(tail) - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str - - def __fspath__(self): - return str(self) - - def as_posix(self): - """Return the string representation of the path with forward (/) - slashes.""" - f = self._flavour - return str(self).replace(f.sep, '/') - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def as_uri(self): - """Return the path as a 'file' URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - return prefix + urlquote_from_bytes(os.fsencode(path)) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - self._str_normcase_cached = self._flavour.normcase(str(self)) - return self._str_normcase_cached - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self._flavour.sep) - return self._parts_normcase_cached - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self._flavour is other._flavour - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __lt__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._load_parts() - return self._drv - - @property - def root(self): - """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._load_parts() - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._load_parts() - return self._tail_cached - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - anchor = self.drive + self.root - return anchor - - @property - def name(self): - """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] - - @property - def suffix(self): - """ - The final component's last suffix, if any. - - This includes the leading period. For example: '.txt' - """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' - - @property - def suffixes(self): - """ - A list of the final component's suffixes, if any. - - These include the leading periods. For example: ['.tar', '.gz'] - """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] - - @property - def stem(self): - """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name - - def with_name(self, name): - """Return a new path with the file name changed.""" - if not self.name: - raise ValueError("%r has an empty name" % (self,)) - f = self._flavour - drv, root, tail = f.splitroot(name) - if drv or root or not tail or f.sep in tail or (f.altsep and f.altsep in tail): - raise ValueError("Invalid name %r" % (name)) - return self._from_parsed_parts(self.drive, self.root, - self._tail[:-1] + [name]) - - def with_stem(self, stem): - """Return a new path with the stem changed.""" - return self.with_name(stem + self.suffix) - - def with_suffix(self, suffix): - """Return a new path with the file suffix changed. If the path - has no suffix, add given suffix. If the given suffix is an empty - string, remove the suffix from the path. - """ - f = self._flavour - if f.sep in suffix or f.altsep and f.altsep in suffix: - raise ValueError("Invalid suffix %r" % (suffix,)) - if suffix and not suffix.startswith('.') or suffix == '.': - raise ValueError("Invalid suffix %r" % (suffix)) - name = self.name - if not name: - raise ValueError("%r has an empty name" % (self,)) - old_suffix = self.suffix - if not old_suffix: - name = name + suffix - else: - name = name[:-len(old_suffix)] + suffix - return self._from_parsed_parts(self.drive, self.root, - self._tail[:-1] + [name]) - - def relative_to(self, other, /, *_deprecated, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, - remove=(3, 14)) - other = self.with_segments(other, *_deprecated) - for step, path in enumerate([other] + list(other.parents)): - if self.is_relative_to(path): - break - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - if step and not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - parts = ['..'] * step + self._tail[len(path._tail):] - return self.with_segments(*parts) - - def is_relative_to(self, other, /, *_deprecated): - """Return True if the path is relative to another path or False. - """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", - msg, remove=(3, 14)) - other = self.with_segments(other, *_deprecated) - return other == self or other in self.parents - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self, *pathsegments) - - def __truediv__(self, key): - try: - return self.joinpath(key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self) - except TypeError: - return NotImplemented - - @property - def parent(self): - """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - return self._from_parsed_parts(drv, root, tail[:-1]) - - @property - def parents(self): - """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - # ntpath.isabs() is defective - see GH-44626 . - if self._flavour is ntpath: - return bool(self.drive and self.root) - return self._flavour.isabs(self._raw_path) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - if self._flavour is posixpath or not self._tail: - return False - - # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not - # exist). We err on the side of caution and return True for paths - # which are not considered reserved by Windows. - if self.drive.startswith('\\\\'): - # UNC paths are never reserved. - return False - name = self._tail[-1].partition('.')[0].partition(':')[0].rstrip(' ') - return name.upper() in _WIN_RESERVED_NAMES - - def match(self, path_pattern): - """ - Return True if this path matches the given pattern. - """ - pat = self.with_segments(path_pattern) - if not pat.parts: - raise ValueError("empty pattern") - pat_parts = pat._parts_normcase - parts = self._parts_normcase - if pat.drive or pat.root: - if len(pat_parts) != len(parts): - return False - elif len(pat_parts) > len(parts): - return False - for part, pat in zip(reversed(parts), reversed(pat_parts)): - if not fnmatch.fnmatchcase(part, pat): - return False - return True - -# Can't subclass os.PathLike from PurePath and keep the constructor -# optimizations in PurePath.__slots__. -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - _flavour = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - _flavour = ntpath - __slots__ = () - - -# Filesystem-accessing classes - - -class Path(PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - - - # Convenience functions for querying the stat results - - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - return self._flavour.ismount(self) - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - return self._flavour.isjunction(self) - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). - """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return self._flavour.samestat(st, other_st) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def read_bytes(self): - """ - Open the file in bytes mode, read it, and close the file. - """ - with self.open(mode='rb') as f: - return f.read() - - def read_text(self, encoding=None, errors=None): - """ - Open the file in text mode, read it, and close the file. - """ - encoding = io.text_encoding(encoding) - with self.open(mode='r', encoding=encoding, errors=errors) as f: - return f.read() - - def write_bytes(self, data): - """ - Open the file in bytes mode, write to it, and close the file. - """ - # type-check for the buffer interface before truncating the file - view = memoryview(data) - with self.open(mode='wb') as f: - return f.write(view) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - if not isinstance(data, str): - raise TypeError('data must be str, not %s' % - data.__class__.__name__) - encoding = io.text_encoding(encoding) - with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: - return f.write(data) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - for name in os.listdir(self): - yield self._make_child_relpath(name) - - def _scandir(self): - # bpo-24132: a future version of pathlib will support subclassing of - # pathlib.Path to customize how the filesystem is accessed. This - # includes scandir(), which is used to implement glob(). - return os.scandir(self) - - def _make_child_relpath(self, name): - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self._flavour.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path - - def glob(self, pattern, *, case_sensitive=None): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - if not pattern: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p - - def rglob(self, pattern, *, case_sensitive=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - paths = [self] - - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. - try: - scandir_it = path._scandir() - except OSError as error: - if on_error is not None: - on_error(error) - continue - - with scandir_it: - dirnames = [] - filenames = [] - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - dirnames.append(entry.name) - else: - filenames.append(entry.name) - - if top_down: - yield path, dirnames, filenames - else: - paths.append((path, dirnames, filenames)) - - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def __enter__(self): - # In previous versions of pathlib, __exit__() marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. - # In Python 3.9 __exit__() was made a no-op. - # In Python 3.11 __enter__() began emitting DeprecationWarning. - # In Python 3.13 __enter__() and __exit__() should be removed. - warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " - "for removal in Python 3.13; Path objects as a context " - "manager is a no-op", - DeprecationWarning, stacklevel=2) - return self - - def __exit__(self, t, v, tb): - pass - - # Public API - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() - - @classmethod - def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). - """ - return cls("~").expanduser() - - def absolute(self): - """Return an absolute version of this path by prepending the current - working directory. No normalization or symlink resolution is performed. - - Use resolve() to get the canonical path to a file. - """ - if self.is_absolute(): - return self - elif self.drive: - # There is a CWD on each drive-letter drive. - cwd = self._flavour.abspath(self.drive) - else: - cwd = os.getcwd() - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - if not self.root and not self._tail: - result = self.with_segments(cwd) - result._str = cwd - return result - return self.with_segments(cwd, self) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - def check_eloop(e): - winerror = getattr(e, 'winerror', 0) - if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: - raise RuntimeError("Symlink loop from %r" % e.filename) - - try: - s = self._flavour.realpath(self, strict=strict) - except OSError as e: - check_eloop(e) - raise - p = self.with_segments(s) - - # In non-strict mode, realpath() doesn't raise on symlink loops. - # Ensure we get an exception by calling stat() - if not strict: - try: - p.stat() - except OSError as e: - check_eloop(e) - return p - - def owner(self): - """ - Return the login name of the file owner. - """ - try: - import pwd - return pwd.getpwuid(self.stat().st_uid).pw_name - except ImportError: - raise NotImplementedError("Path.owner() is unsupported on this system") - - def group(self): - """ - Return the group name of the file gid. - """ - - try: - import grp - return grp.getgrgid(self.stat().st_gid).gr_name - except ImportError: - raise NotImplementedError("Path.group() is unsupported on this system") - - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - if not hasattr(os, "readlink"): - raise NotImplementedError("os.readlink() not available on this system") - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - if not hasattr(os, "symlink"): - raise NotImplementedError("os.symlink() not available on this system") - os.symlink(target, self, target_is_directory) - - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - if not hasattr(os, "link"): - raise NotImplementedError("os.link() not available on this system") - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = self._flavour.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise NotImplementedError( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise NotImplementedError( - f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py new file mode 100644 index 00000000000000..f4668ab3270e51 --- /dev/null +++ b/Lib/pathlib/__init__.py @@ -0,0 +1,507 @@ +"""Object-oriented filesystem paths. + +This module provides classes to represent abstract paths and concrete +paths with operations that have semantics appropriate for different +operating systems. +""" + +import io +import ntpath +import os +import posixpath + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from . import _abc + + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +UnsupportedOperation = _abc.UnsupportedOperation + + +class PurePath(_abc.PurePathBase): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.pathmod is ntpath and self.pathmod is posixpath: + # GH-103631: Convert separators for backwards compatibility. + paths.extend(path.replace('\\', '/') for path in arg._raw_paths) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + super().__init__(*paths) + + def __reduce__(self): + # Using the parts tuple helps share interned path parts + # when pickling related paths. + return (self.__class__, self.parts) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if _abc._is_case_sensitive(self.pathmod): + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + pathmod = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + pathmod = ntpath + __slots__ = () + + +class Path(_abc.PathBase, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + as_uri = PurePath.as_uri + + @classmethod + def _unsupported(cls, method_name): + msg = f"{cls.__name__}.{method_name}() is unsupported on this system" + raise UnsupportedOperation(msg) + + def __init__(self, *args, **kwargs): + if kwargs: + import warnings + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return (self._make_child_relpath(name) for name in os.listdir(self)) + + def _scandir(self): + return os.scandir(self) + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + result = self.with_segments(cwd) + result._str = cwd + return result + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.pathmod.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py new file mode 100644 index 00000000000000..4808d0e61f7038 --- /dev/null +++ b/Lib/pathlib/_abc.py @@ -0,0 +1,1150 @@ +import functools +import io +import ntpath +import os +import posixpath +import sys +import warnings +from _collections_abc import Sequence +from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL +from itertools import chain +from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO + +# +# Internals +# + +# Maximum number of symlinks to follow in PathBase.resolve() +_MAX_SYMLINKS = 40 + +# Reference for Windows paths can be found at +# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . +_WIN_RESERVED_NAMES = frozenset( + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | + {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} +) + +_WINERROR_NOT_READY = 21 # drive exists but is not accessible +_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 +_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself + +# EBADF - guard against macOS `stat` throwing EBADF +_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) + +_IGNORED_WINERRORS = ( + _WINERROR_NOT_READY, + _WINERROR_INVALID_NAME, + _WINERROR_CANT_RESOLVE_FILENAME) + +def _ignore_error(exception): + return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or + getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) + + +@functools.cache +def _is_case_sensitive(pathmod): + return pathmod.normcase('Aa') == 'Aa' + +# +# Globbing helpers +# + +re = glob = None + + +@functools.lru_cache(maxsize=256) +def _compile_pattern(pat, sep, case_sensitive): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + global re, glob + if re is None: + import re, glob + + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = glob.translate(pat, recursive=True, include_hidden=True, seps=sep) + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we consume it with an atomic group. + regex = r'(\.\Z)?+' + regex + return re.compile(regex, flags=flags).match + + +def _select_children(parent_paths, dir_only, follow_symlinks, match): + """Yield direct children of given paths, filtering by name and type.""" + if follow_symlinks is None: + follow_symlinks = True + for parent_path in parent_paths: + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with parent_path._scandir() as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + if dir_only: + try: + if not entry.is_dir(follow_symlinks=follow_symlinks): + continue + except OSError: + continue + name = entry.name + if match(name): + yield parent_path._make_child_relpath(name) + + +def _select_recursive(parent_paths, dir_only, follow_symlinks): + """Yield given paths and all their subdirectories, recursively.""" + if follow_symlinks is None: + follow_symlinks = False + for parent_path in parent_paths: + paths = [parent_path] + while paths: + path = paths.pop() + yield path + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with path._scandir() as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + paths.append(path._make_child_relpath(entry.name)) + continue + except OSError: + pass + if not dir_only: + yield path._make_child_relpath(entry.name) + + +def _select_unique(paths): + """Yields the given paths, filtering out duplicates.""" + yielded = set() + try: + for path in paths: + path_str = str(path) + if path_str not in yielded: + yield path + yielded.add(path_str) + finally: + yielded.clear() + + +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is called on + a path object. + """ + pass + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePathBase: + """Base class for pure path objects. + + This class *does not* provide several magic methods that are defined in + its subclass PurePath. They are: __fspath__, __bytes__, __reduce__, + __hash__, __eq__, __lt__, __le__, __gt__, __ge__. Its initializer and path + joining methods accept only strings, not os.PathLike objects more broadly. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unnormalized string paths. This is set + # in the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The '_resolving' slot stores a boolean indicating whether the path + # is being processed by `PathBase.resolve()`. This prevents duplicate + # work from occurring when `resolve()` calls `stat()` or `readlink()`. + '_resolving', + ) + pathmod = os.path + + def __init__(self, *paths): + self._raw_paths = paths + self._resolving = False + + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.pathmod.sep + altsep = cls.pathmod.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.pathmod.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] + return drv, root, parsed + + def _load_parts(self): + paths = self._raw_paths + if len(paths) == 0: + path = '' + elif len(paths) == 1: + path = paths[0] + else: + path = self.pathmod.join(*paths) + drv, root, tail = self._parse_path(path) + self._drv = drv + self._root = root + self._tail_cached = tail + + def _from_parsed_parts(self, drv, root, tail): + path_str = self._format_parsed_parts(drv, root, tail) + path = self.with_segments(path_str) + path._str = path_str or '.' + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.pathmod.sep.join(tail) + elif tail and cls.pathmod.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.pathmod.sep.join(tail) + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.pathmod.sep, '/') + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._load_parts() + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._load_parts() + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._load_parts() + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + anchor = self.drive + self.root + return anchor + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name + i = name.rfind('.') + if 0 < i < len(name) - 1: + return name[i:] + else: + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + name = self.name + if name.endswith('.'): + return [] + name = name.lstrip('.') + return ['.' + suffix for suffix in name.split('.')[1:]] + + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if 0 < i < len(name) - 1: + return name[:i] + else: + return name + + def with_name(self, name): + """Return a new path with the file name changed.""" + m = self.pathmod + if not name or m.sep in name or (m.altsep and m.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + return self.with_name(stem + self.suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + if not suffix: + return self.with_name(self.stem) + elif suffix.startswith('.') and len(suffix) > 1: + return self.with_name(self.stem + suffix) + else: + raise ValueError(f"Invalid suffix {suffix!r}") + + def relative_to(self, other, /, *_deprecated, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if _deprecated: + msg = ("support for supplying more than one positional argument " + "to pathlib.PurePath.relative_to() is deprecated and " + "scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, + remove=(3, 14)) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePathBase): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other, /, *_deprecated): + """Return True if the path is relative to another path or False. + """ + if _deprecated: + msg = ("support for supplying more than one argument to " + "pathlib.PurePath.is_relative_to() is deprecated and " + "scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", + msg, remove=(3, 14)) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePathBase): + other = self.with_segments(other) + return other == self or other in self.parents + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(*self._raw_paths, *pathsegments) + + def __truediv__(self, key): + try: + return self.joinpath(key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, *self._raw_paths) + except TypeError: + return NotImplemented + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + path = self._from_parsed_parts(drv, root, tail[:-1]) + path._resolving = self._resolving + return path + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.pathmod is ntpath: + # ntpath.isabs() is defective - see GH-44626. + return bool(self.drive and self.root) + elif self.pathmod is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + else: + return self.pathmod.isabs(str(self)) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + if self.pathmod is posixpath or not self._tail: + return False + + # NOTE: the rules for reserved names seem somewhat complicated + # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not + # exist). We err on the side of caution and return True for paths + # which are not considered reserved by Windows. + if self.drive.startswith('\\\\'): + # UNC paths are never reserved. + return False + name = self._tail[-1].partition('.')[0].partition(':')[0].rstrip(' ') + return name.upper() in _WIN_RESERVED_NAMES + + def match(self, path_pattern, *, case_sensitive=None): + """ + Return True if this path matches the given pattern. + """ + if not isinstance(path_pattern, PurePathBase): + path_pattern = self.with_segments(path_pattern) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.pathmod) + sep = path_pattern.pathmod.sep + pattern_str = str(path_pattern) + if path_pattern.drive or path_pattern.root: + pass + elif path_pattern._tail: + pattern_str = f'**{sep}{pattern_str}' + else: + raise ValueError("empty pattern") + match = _compile_pattern(pattern_str, sep, case_sensitive) + return match(str(self)) is not None + + + +class PathBase(PurePathBase): + """Base class for concrete path objects. + + This class provides dummy implementations for many methods that derived + classes can override selectively; the default implementations raise + UnsupportedOperation. The most basic methods, such as stat() and open(), + directly raise UnsupportedOperation; these basic methods are called by + other methods such as is_dir() and read_text(). + + The Path class derives this class to implement local filesystem paths. + Users may derive their own classes to implement virtual filesystem paths, + such as paths in archive files or on remote storage systems. + """ + __slots__ = () + + @classmethod + def _unsupported(cls, method_name): + msg = f"{cls.__name__}.{method_name}() is unsupported" + raise UnsupportedOperation(msg) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + self._unsupported("stat") + + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return self.stat(follow_symlinks=False) + + + # Convenience functions for querying the stat results + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + try: + self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if not _ignore_error(e): + raise + return False + except ValueError: + # Non-encodable path + return False + return True + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_mount(self): + """ + Check if this path is a mount point + """ + # Need to exist and be a dir + if not self.exists() or not self.is_dir(): + return False + + try: + parent_dev = self.parent.stat().st_dev + except OSError: + return False + + dev = self.stat().st_dev + if dev != parent_dev: + return True + ino = self.stat().st_ino + parent_ino = self.parent.stat().st_ino + return ino == parent_ino + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + try: + return S_ISLNK(self.lstat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist + return False + except ValueError: + # Non-encodable path + return False + + def is_junction(self): + """ + Whether this path is a junction. + """ + # Junctions are a Windows-only feature, not present in POSIX nor the + # majority of virtual filesystems. There is no cross-platform idiom + # to check for junctions (using stat().st_mode). + return False + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + self._unsupported("open") + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb') as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + encoding = io.text_encoding(encoding) + with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with self.open(mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + encoding = io.text_encoding(encoding) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + self._unsupported("iterdir") + + def _scandir(self): + # Emulate os.scandir(), which returns an object that can be used as a + # context manager. This method is called by walk() and glob(). + from contextlib import nullcontext + return nullcontext(self.iterdir()) + + def _make_child_relpath(self, name): + path_str = str(self) + tail = self._tail + if tail: + path_str = f'{path_str}{self.pathmod.sep}{name}' + elif path_str != '.': + path_str = f'{path_str}{name}' + else: + path_str = name + path = self.with_segments(path_str) + path._str = path_str + path._drv = self.drive + path._root = self.root + path._tail_cached = tail + [name] + return path + + def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + return self._glob(pattern, case_sensitive, follow_symlinks) + + def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + return self._glob(f'**/{pattern}', case_sensitive, follow_symlinks) + + def _glob(self, pattern, case_sensitive, follow_symlinks): + path_pattern = self.with_segments(pattern) + if path_pattern.drive or path_pattern.root: + raise NotImplementedError("Non-relative patterns are unsupported") + elif not path_pattern._tail: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + + pattern_parts = path_pattern._tail.copy() + if pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): + # GH-65238: pathlib doesn't preserve trailing slash. Add it back. + pattern_parts.append('') + if pattern_parts[-1] == '**': + # GH-70303: '**' only matches directories. Add trailing slash. + warnings.warn( + "Pattern ending '**' will match files and directories in a " + "future Python release. Add a trailing slash to match only " + "directories and remove this warning.", + FutureWarning, 3) + pattern_parts.append('') + + if case_sensitive is None: + # TODO: evaluate case-sensitivity of each directory in _select_children(). + case_sensitive = _is_case_sensitive(self.pathmod) + + # If symlinks are handled consistently, and the pattern does not + # contain '..' components, then we can use a 'walk-and-match' strategy + # when expanding '**' wildcards. When a '**' wildcard is encountered, + # all following pattern parts are immediately consumed and used to + # build a `re.Pattern` object. This pattern is used to filter the + # recursive walk. As a result, pattern parts following a '**' wildcard + # do not perform any filesystem access, which can be much faster! + filter_paths = follow_symlinks is not None and '..' not in pattern_parts + deduplicate_paths = False + sep = self.pathmod.sep + paths = iter([self] if self.is_dir() else []) + part_idx = 0 + while part_idx < len(pattern_parts): + part = pattern_parts[part_idx] + part_idx += 1 + if part == '': + # Trailing slash. + pass + elif part == '..': + paths = (path._make_child_relpath('..') for path in paths) + elif part == '**': + # Consume adjacent '**' components. + while part_idx < len(pattern_parts) and pattern_parts[part_idx] == '**': + part_idx += 1 + + if filter_paths and part_idx < len(pattern_parts) and pattern_parts[part_idx] != '': + dir_only = pattern_parts[-1] == '' + paths = _select_recursive(paths, dir_only, follow_symlinks) + + # Filter out paths that don't match pattern. + prefix_len = len(str(self._make_child_relpath('_'))) - 1 + match = _compile_pattern(str(path_pattern), sep, case_sensitive) + paths = (path for path in paths if match(str(path), prefix_len)) + return paths + + dir_only = part_idx < len(pattern_parts) + paths = _select_recursive(paths, dir_only, follow_symlinks) + if deduplicate_paths: + # De-duplicate if we've already seen a '**' component. + paths = _select_unique(paths) + deduplicate_paths = True + elif '**' in part: + raise ValueError("Invalid pattern: '**' can only be an entire path component") + else: + dir_only = part_idx < len(pattern_parts) + match = _compile_pattern(part, sep, case_sensitive) + paths = _select_children(paths, dir_only, follow_symlinks, match) + return paths + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + paths = [self] + + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + + # We may not have read permission for self, in which case we can't + # get a list of the files the directory contains. os.walk() + # always suppressed the exception in that instance, rather than + # blow up for a minor reason when (say) a thousand readable + # directories are still left to visit. That logic is copied here. + try: + scandir_obj = path._scandir() + except OSError as error: + if on_error is not None: + on_error(error) + continue + + with scandir_obj as scandir_it: + dirnames = [] + filenames = [] + for entry in scandir_it: + try: + is_dir = entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + # Carried over from os.path.isdir(). + is_dir = False + + if is_dir: + dirnames.append(entry.name) + else: + filenames.append(entry.name) + + if top_down: + yield path, dirnames, filenames + else: + paths.append((path, dirnames, filenames)) + + paths += [path._make_child_relpath(d) for d in reversed(dirnames)] + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + self._unsupported("absolute") + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + # We call 'absolute()' rather than using 'os.getcwd()' directly to + # enable users to replace the implementation of 'absolute()' in a + # subclass and benefit from the new behaviour here. This works because + # os.path.abspath('.') == os.getcwd(). + return cls().absolute() + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + self._unsupported("expanduser") + + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + return cls("~").expanduser() + + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + self._unsupported("readlink") + readlink._supported = False + + def _split_stack(self): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + if not self._tail: + return self, [] + return self._from_parsed_parts(self.drive, self.root, []), self._tail[::-1] + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + if self._resolving: + return self + path, parts = self._split_stack() + try: + path = path.absolute() + except UnsupportedOperation: + pass + + # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported + # and (in non-strict mode) we can improve performance by not calling `stat()`. + querying = strict or getattr(self.readlink, '_supported', True) + link_count = 0 + while parts: + part = parts.pop() + if part == '..': + if not path._tail: + if path.root: + # Delete '..' segment immediately following root + continue + elif path._tail[-1] != '..': + # Delete '..' segment and its predecessor + path = path.parent + continue + next_path = path._make_child_relpath(part) + if querying and part != '..': + next_path._resolving = True + try: + st = next_path.stat(follow_symlinks=False) + if S_ISLNK(st.st_mode): + # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are + # encountered during resolution. + link_count += 1 + if link_count >= _MAX_SYMLINKS: + raise OSError(ELOOP, "Too many symbolic links in path", str(self)) + target, target_parts = next_path.readlink()._split_stack() + # If the symlink target is absolute (like '/etc/hosts'), set the current + # path to its uppermost parent (like '/'). + if target.root: + path = target + # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to + # the stack of unresolved path parts. + parts.extend(target_parts) + continue + elif parts and not S_ISDIR(st.st_mode): + raise NotADirectoryError(ENOTDIR, "Not a directory", str(self)) + except OSError: + if strict: + raise + else: + querying = False + next_path._resolving = False + path = next_path + return path + + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + self._unsupported("symlink_to") + + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + self._unsupported("hardlink_to") + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + self._unsupported("touch") + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + self._unsupported("mkdir") + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + self._unsupported("rename") + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + self._unsupported("replace") + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + self._unsupported("chmod") + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + self._unsupported("unlink") + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + self._unsupported("rmdir") + + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + self._unsupported("owner") + + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + self._unsupported("group") + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + cls._unsupported("from_uri") + + def as_uri(self): + """Return the path as a URI.""" + self._unsupported("as_uri") diff --git a/Lib/pdb.py b/Lib/pdb.py index 6b6feac1ddead1..83b7fefec63636 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -76,6 +76,7 @@ import dis import code import glob +import codeop import pprint import signal import inspect @@ -84,6 +85,7 @@ import traceback import linecache +from contextlib import contextmanager from typing import Union @@ -136,9 +138,14 @@ def check(self): if not os.path.exists(self): print('Error:', self.orig, 'does not exist') sys.exit(1) + if os.path.isdir(self): + print('Error:', self.orig, 'is a directory') + sys.exit(1) - # Replace pdb's dir with script's dir in front of module search path. - sys.path[0] = os.path.dirname(self) + # If safe_path(-P) is not set, sys.path[0] is the directory + # of pdb, and we should replace it with the directory of the script + if not sys.flags.safe_path: + sys.path[0] = os.path.dirname(self) @property def filename(self): @@ -162,6 +169,9 @@ class _ModuleTarget(str): def check(self): try: self._details + except ImportError as e: + print(f"ImportError: {e}") + sys.exit(1) except Exception: traceback.print_exc() sys.exit(1) @@ -197,6 +207,15 @@ def namespace(self): ) +class _PdbInteractiveConsole(code.InteractiveConsole): + def __init__(self, ns, message): + self._message = message + super().__init__(locals=ns, local_exit=True) + + def write(self, data): + self._message(data, end='') + + # Interaction prompt line will separate file and call info from code # text using value of line_prefix string. A newline and arrow may # be to your liking. You can set it once pdb is imported using the @@ -204,10 +223,15 @@ def namespace(self): # line_prefix = ': ' # Use this to get the old situation back line_prefix = '\n-> ' # Probably a better default -class Pdb(bdb.Bdb, cmd.Cmd): + +class Pdb(bdb.Bdb, cmd.Cmd): _previous_sigint_handler = None + # Limit the maximum depth of chained exceptions, we should be handling cycles, + # but in case there are recursions, we stop at 999. + MAX_CHAINED_EXCEPTION_DEPTH = 999 + def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True): bdb.Bdb.__init__(self, skip=skip) @@ -225,11 +249,14 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, try: import readline # remove some common file name delimiters - readline.set_completer_delims(' \t\n`@#$%^&*()=+[{]}\\|;:\'",<>?') + readline.set_completer_delims(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') except ImportError: pass self.allow_kbdint = False self.nosigint = nosigint + # Consider these characters as part of the command so when the users type + # c.a or c['a'], it won't be recognized as a c(ontinue) command + self.identchars = cmd.Cmd.identchars + '=.[](),"\'+-*/%@&|<>~^' # Read ~/.pdbrc and ./.pdbrc self.rcLines = [] @@ -255,6 +282,9 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.commands_bnum = None # The breakpoint number for which we are # defining a list + self._chained_exceptions = tuple() + self._chained_exception_index = 0 + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt @@ -291,6 +321,14 @@ def setup(self, f, tb): # cache it here to ensure that modifications are not overwritten. self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) + + if self._chained_exceptions: + self.set_convenience_variable( + self.curframe, + '_exception', + self._chained_exceptions[self._chained_exception_index], + ) + return self.execRcLines() # Can be executed earlier than 'setup' if desired @@ -410,10 +448,68 @@ def preloop(self): # fields are changed to be displayed if newvalue is not oldvalue and newvalue != oldvalue: displaying[expr] = newvalue - self.message('display %s: %r [old: %r]' % - (expr, newvalue, oldvalue)) + self.message('display %s: %s [old: %s]' % + (expr, self._safe_repr(newvalue, expr), + self._safe_repr(oldvalue, expr))) + + def _get_tb_and_exceptions(self, tb_or_exc): + """ + Given a tracecack or an exception, return a tuple of chained exceptions + and current traceback to inspect. - def interaction(self, frame, traceback): + This will deal with selecting the right ``__cause__`` or ``__context__`` + as well as handling cycles, and return a flattened list of exceptions we + can jump to with do_exceptions. + + """ + _exceptions = [] + if isinstance(tb_or_exc, BaseException): + traceback, current = tb_or_exc.__traceback__, tb_or_exc + + while current is not None: + if current in _exceptions: + break + _exceptions.append(current) + if current.__cause__ is not None: + current = current.__cause__ + elif ( + current.__context__ is not None and not current.__suppress_context__ + ): + current = current.__context__ + + if len(_exceptions) >= self.MAX_CHAINED_EXCEPTION_DEPTH: + self.message( + f"More than {self.MAX_CHAINED_EXCEPTION_DEPTH}" + " chained exceptions found, not all exceptions" + "will be browsable with `exceptions`." + ) + break + else: + traceback = tb_or_exc + return tuple(reversed(_exceptions)), traceback + + @contextmanager + def _hold_exceptions(self, exceptions): + """ + Context manager to ensure proper cleaning of exceptions references + + When given a chained exception instead of a traceback, + pdb may hold references to many objects which may leak memory. + + We use this context manager to make sure everything is properly cleaned + + """ + try: + self._chained_exceptions = exceptions + self._chained_exception_index = len(exceptions) - 1 + yield + finally: + # we can't put those in forget as otherwise they would + # be cleared on exception change + self._chained_exceptions = tuple() + self._chained_exception_index = 0 + + def interaction(self, frame, tb_or_exc): # Restore the previous signal handler at the Pdb prompt. if Pdb._previous_sigint_handler: try: @@ -422,14 +518,19 @@ def interaction(self, frame, traceback): pass else: Pdb._previous_sigint_handler = None - if self.setup(frame, traceback): - # no interaction desired at this time (happens if .pdbrc contains - # a command like "continue") + + _chained_exceptions, tb = self._get_tb_and_exceptions(tb_or_exc) + if isinstance(tb_or_exc, BaseException): + assert tb is not None, "main exception must have a traceback" + with self._hold_exceptions(_chained_exceptions): + if self.setup(frame, tb): + # no interaction desired at this time (happens if .pdbrc contains + # a command like "continue") + self.forget() + return + self.print_stack_entry(self.stack[self.curindex]) + self._cmdloop() self.forget() - return - self.print_stack_entry(self.stack[self.curindex]) - self._cmdloop() - self.forget() def displayhook(self, obj): """Custom displayhook for the exec in default(), which prevents @@ -439,12 +540,52 @@ def displayhook(self, obj): if obj is not None: self.message(repr(obj)) + @contextmanager + def _disable_tab_completion(self): + if self.use_rawinput and self.completekey == 'tab': + try: + import readline + except ImportError: + yield + return + try: + readline.parse_and_bind('tab: self-insert') + yield + finally: + readline.parse_and_bind('tab: complete') + else: + yield + def default(self, line): if line[:1] == '!': line = line[1:].strip() locals = self.curframe_locals globals = self.curframe.f_globals try: - code = compile(line + '\n', '', 'single') + if (code := codeop.compile_command(line + '\n', '', 'single')) is None: + # Multi-line mode + with self._disable_tab_completion(): + buffer = line + continue_prompt = "... " + while (code := codeop.compile_command(buffer, '', 'single')) is None: + if self.use_rawinput: + try: + line = input(continue_prompt) + except (EOFError, KeyboardInterrupt): + self.lastcmd = "" + print('\n') + return + else: + self.stdout.write(continue_prompt) + self.stdout.flush() + line = self.stdin.readline() + if not len(line): + self.lastcmd = "" + self.stdout.write('\n') + self.stdout.flush() + return + else: + line = line.rstrip('\r\n') + buffer += '\n' + line save_stdout = sys.stdout save_stdin = sys.stdin save_displayhook = sys.displayhook @@ -467,11 +608,20 @@ def precmd(self, line): args = line.split() while args[0] in self.aliases: line = self.aliases[args[0]] - ii = 1 - for tmpArg in args[1:]: - line = line.replace("%" + str(ii), - tmpArg) - ii += 1 + for idx in range(1, 10): + if f'%{idx}' in line: + if idx >= len(args): + self.error(f"Not enough arguments for alias '{args[0]}'") + # This is a no-op + return "!" + line = line.replace(f'%{idx}', args[idx]) + elif '%*' not in line: + if idx < len(args): + self.error(f"Too many arguments for alias '{args[0]}'") + # This is a no-op + return "!" + break + line = line.replace("%*", ' '.join(args[1:])) args = line.split() # split into ';;' separated commands @@ -486,6 +636,7 @@ def precmd(self, line): # Replace all the convenience variables line = re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', r'__pdb_convenience_variables["\1"]', line) + return line def onecmd(self, line): @@ -530,8 +681,8 @@ def handle_command_def(self, line): # interface abstraction functions - def message(self, msg): - print(msg, file=self.stdout) + def message(self, msg, end='\n'): + print(msg, end=end, file=self.stdout) def error(self, msg): print('***', msg, file=self.stdout) @@ -546,6 +697,18 @@ def set_convenience_variable(self, frame, name, value): # Generic completion functions. Individual complete_foo methods can be # assigned below to one of these functions. + def completenames(self, text, line, begidx, endidx): + # Overwrite completenames() of cmd so for the command completion, + # if no current command matches, check for expressions as well + commands = super().completenames(text, line, begidx, endidx) + for alias in self.aliases: + if alias.startswith(text): + commands.append(alias) + if commands: + return commands + else: + return self._complete_expression(text, line, begidx, endidx) + def _complete_location(self, text, line, begidx, endidx): # Complete a file/module/function location for break/tbreak/clear. if line.strip().endswith((':', ',')): @@ -580,6 +743,10 @@ def _complete_expression(self, text, line, begidx, endidx): # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. ns = {**self.curframe.f_globals, **self.curframe_locals} + if text.startswith("$"): + # Complete convenience variables + conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) + return [f"${name}" for name in conv_vars if name.startswith(text[1:])] if '.' in text: # Walk an attribute chain up to the last part, similar to what # rlcompleter does. This will bail if any of the parts are not @@ -644,7 +811,7 @@ def do_commands(self, arg): try: bnum = int(arg) except: - self.error("Usage: commands [bnum]\n ...\n end") + self._print_invalid_arg(arg) return try: self.get_bpbynumber(bnum) @@ -757,7 +924,7 @@ def do_break(self, arg, temporary = 0): #use co_name to identify the bkpt (function names #could be aliased, but co_name is invariant) funcname = code.co_name - lineno = code.co_firstlineno + lineno = self._find_first_executable_line(code) filename = code.co_filename except: # last thing to try @@ -860,6 +1027,23 @@ def checkline(self, filename, lineno): return 0 return lineno + def _find_first_executable_line(self, code): + """ Try to find the first executable line of the code object. + + Equivalently, find the line number of the instruction that's + after RESUME + + Return code.co_firstlineno if no executable line is found. + """ + prev = None + for instr in dis.get_instructions(code): + if prev is not None and prev.opname == 'RESUME': + if instr.positions.lineno is not None: + return instr.positions.lineno + return code.co_firstlineno + prev = instr + return code.co_firstlineno + def do_enable(self, arg): """enable bpnumber [bpnumber ...] @@ -941,14 +1125,22 @@ def do_ignore(self, arg): condition evaluates to true. """ args = arg.split() - try: - count = int(args[1].strip()) - except: + if not args: + self.error('Breakpoint number expected') + return + if len(args) == 1: count = 0 + elif len(args) == 2: + try: + count = int(args[1]) + except ValueError: + self._print_invalid_arg(arg) + return + else: + self._print_invalid_arg(arg) + return try: bp = self.get_bpbynumber(args[0].strip()) - except IndexError: - self.error('Breakpoint number expected') except ValueError as err: self.error(err) else: @@ -1025,6 +1217,9 @@ def do_where(self, arg): An arrow indicates the "current frame", which determines the context of most commands. 'bt' is an alias for this command. """ + if arg: + self._print_invalid_arg(arg) + return self.print_stack_trace() do_w = do_where do_bt = do_where @@ -1038,6 +1233,53 @@ def _select_frame(self, number): self.print_stack_entry(self.stack[self.curindex]) self.lineno = None + def do_exceptions(self, arg): + """exceptions [number] + + List or change current exception in an exception chain. + + Without arguments, list all the current exception in the exception + chain. Exceptions will be numbered, with the current exception indicated + with an arrow. + + If given an integer as argument, switch to the exception at that index. + """ + if not self._chained_exceptions: + self.message( + "Did not find chained exceptions. To move between" + " exceptions, pdb/post_mortem must be given an exception" + " object rather than a traceback." + ) + return + if not arg: + for ix, exc in enumerate(self._chained_exceptions): + prompt = ">" if ix == self._chained_exception_index else " " + rep = repr(exc) + if len(rep) > 80: + rep = rep[:77] + "..." + indicator = ( + " -" + if self._chained_exceptions[ix].__traceback__ is None + else f"{ix:>3}" + ) + self.message(f"{prompt} {indicator} {rep}") + else: + try: + number = int(arg) + except ValueError: + self.error("Argument must be an integer") + return + if 0 <= number < len(self._chained_exceptions): + if self._chained_exceptions[number].__traceback__ is None: + self.error("This exception does not have a traceback, cannot jump to it") + return + + self._chained_exception_index = number + self.setup(None, self._chained_exceptions[number].__traceback__) + self.print_stack_entry(self.stack[self.curindex]) + else: + self.error("No exception with that number") + def do_up(self, arg): """u(p) [count] @@ -1112,6 +1354,9 @@ def do_step(self, arg): (either in a function that is called or in the current function). """ + if arg: + self._print_invalid_arg(arg) + return self.set_step() return 1 do_s = do_step @@ -1122,6 +1367,9 @@ def do_next(self, arg): Continue execution until the next line in the current function is reached or it returns. """ + if arg: + self._print_invalid_arg(arg) + return self.set_next(self.curframe) return 1 do_n = do_next @@ -1153,6 +1401,9 @@ def do_return(self, arg): Continue execution until the current function returns. """ + if arg: + self._print_invalid_arg(arg) + return self.set_return(self.curframe) return 1 do_r = do_return @@ -1162,6 +1413,9 @@ def do_continue(self, arg): Continue execution, only stop when a breakpoint is encountered. """ + if arg: + self._print_invalid_arg(arg) + return if not self.nosigint: try: Pdb._previous_sigint_handler = \ @@ -1256,6 +1510,9 @@ def do_args(self, arg): Print the argument list of the current function. """ + if arg: + self._print_invalid_arg(arg) + return co = self.curframe.f_code dict = self.curframe_locals n = co.co_argcount + co.co_kwonlyargcount @@ -1264,7 +1521,7 @@ def do_args(self, arg): for i in range(n): name = co.co_varnames[i] if name in dict: - self.message('%s = %r' % (name, dict[name])) + self.message('%s = %s' % (name, self._safe_repr(dict[name], name))) else: self.message('%s = *** undefined ***' % (name,)) do_a = do_args @@ -1274,8 +1531,11 @@ def do_retval(self, arg): Print the return value for the last return of a function. """ + if arg: + self._print_invalid_arg(arg) + return if '__return__' in self.curframe_locals: - self.message(repr(self.curframe_locals['__return__'])) + self.message(self._safe_repr(self.curframe_locals['__return__'], "retval")) else: self.error('Not yet returned!') do_rv = do_retval @@ -1310,6 +1570,12 @@ def _msg_val_func(self, arg, func): except: self._error_exc() + def _safe_repr(self, obj, expr): + try: + return repr(obj) + except Exception as e: + return _rstr(f"*** repr({expr}) failed: {self._format_exc(e)} ***") + def do_p(self, arg): """p expression @@ -1390,6 +1656,9 @@ def do_longlist(self, arg): List the whole source code for the current function or frame. """ + if arg: + self._print_invalid_arg(arg) + return filename = self.curframe.f_code.co_filename breaklist = self.get_file_breaks(filename) try: @@ -1486,8 +1755,8 @@ def do_display(self, arg): if not arg: if self.displaying: self.message('Currently displaying:') - for item in self.displaying.get(self.curframe, {}).items(): - self.message('%s: %r' % item) + for key, val in self.displaying.get(self.curframe, {}).items(): + self.message('%s: %s' % (key, self._safe_repr(val, key))) else: self.message('No expression is being displayed') else: @@ -1496,7 +1765,7 @@ def do_display(self, arg): else: val = self._getval_except(arg) self.displaying.setdefault(self.curframe, {})[arg] = val - self.message('display %s: %r' % (arg, val)) + self.message('display %s: %s' % (arg, self._safe_repr(val, arg))) complete_display = _complete_expression @@ -1526,7 +1795,9 @@ def do_interact(self, arg): contains all the (global and local) names found in the current scope. """ ns = {**self.curframe.f_globals, **self.curframe_locals} - code.interact("*interactive*", local=ns) + console = _PdbInteractiveConsole(ns, message=self.message) + console.interact(banner="*pdb interact start*", + exitmsg="*exit from pdb interact command*") def do_alias(self, arg): """alias [name [command]] @@ -1559,10 +1830,24 @@ def do_alias(self, arg): for alias in keys: self.message("%s = %s" % (alias, self.aliases[alias])) return - if args[0] in self.aliases and len(args) == 1: - self.message("%s = %s" % (args[0], self.aliases[args[0]])) + if len(args) == 1: + if args[0] in self.aliases: + self.message("%s = %s" % (args[0], self.aliases[args[0]])) + else: + self.error(f"Unknown alias '{args[0]}'") else: - self.aliases[args[0]] = ' '.join(args[1:]) + # Do a validation check to make sure no replaceable parameters + # are skipped if %* is not used. + alias = ' '.join(args[1:]) + if '%*' not in alias: + consecutive = True + for idx in range(1, 10): + if f'%{idx}' not in alias: + consecutive = False + if f'%{idx}' in alias and not consecutive: + self.error("Replaceable parameters must be consecutive") + return + self.aliases[args[0]] = alias def do_unalias(self, arg): """unalias name @@ -1570,7 +1855,9 @@ def do_unalias(self, arg): Delete the specified alias. """ args = arg.split() - if len(args) == 0: return + if len(args) == 0: + self._print_invalid_arg(arg) + return if args[0] in self.aliases: del self.aliases[args[0]] @@ -1723,7 +2010,7 @@ def _getsourcelines(self, obj): lineno = max(1, lineno) return lines, lineno - def _help_message_from_doc(self, doc): + def _help_message_from_doc(self, doc, usage_only=False): lines = [line.strip() for line in doc.rstrip().splitlines()] if not lines: return "No help message found." @@ -1739,10 +2026,24 @@ def _help_message_from_doc(self, doc): elif i < usage_end: prefix = " " else: + if usage_only: + break prefix = "" formatted.append(indent + prefix + line) return "\n".join(formatted) + def _print_invalid_arg(self, arg): + """Return the usage string for a function.""" + + self.error(f"Invalid argument: {arg}") + + # Yes it's a bit hacky. Get the caller name, get the method based on + # that name, and get the docstring from that method. + # This should NOT fail if the caller is a method of this class. + doc = inspect.getdoc(getattr(self, sys._getframe(1).f_code.co_name)) + if doc is not None: + self.message(self._help_message_from_doc(doc, usage_only=True)) + # Collect all command help into docstring, if not run with -OO if __doc__ is not None: @@ -1818,11 +2119,23 @@ def set_trace(*, header=None): # Post-Mortem interface def post_mortem(t=None): - """Enter post-mortem debugging of the given *traceback* object. + """Enter post-mortem debugging of the given *traceback*, or *exception* + object. If no traceback is given, it uses the one of the exception that is currently being handled (an exception must be being handled if the default is to be used). + + If `t` is an exception object, the `exceptions` command makes it possible to + list and inspect its chained exceptions (if any). + """ + return _post_mortem(t, Pdb()) + + +def _post_mortem(t, pdb_instance): + """ + Private version of post_mortem, which allow to pass a pdb instance + for testing purposes. """ # handling the default if t is None: @@ -1830,21 +2143,17 @@ def post_mortem(t=None): if exc is not None: t = exc.__traceback__ - if t is None: + if t is None or (isinstance(t, BaseException) and t.__traceback__ is None): raise ValueError("A valid traceback must be passed if no " "exception is being handled") - p = Pdb() - p.reset() - p.interaction(None, t) + pdb_instance.reset() + pdb_instance.interaction(None, t) + def pm(): - """Enter post-mortem debugging of the traceback found in sys.last_traceback.""" - if hasattr(sys, 'last_exc'): - tb = sys.last_exc.__traceback__ - else: - tb = sys.last_traceback - post_mortem(tb) + """Enter post-mortem debugging of the traceback found in sys.last_exc.""" + post_mortem(sys.last_exc) # Main program for testing @@ -1860,8 +2169,6 @@ def help(): pydoc.pager(__doc__) _usage = """\ -usage: pdb.py [-c command] ... [-m module | pyfile] [arg] ... - Debug the Python program given by pyfile. Alternatively, an executable module or package to debug can be specified using the -m switch. @@ -1876,40 +2183,47 @@ def help(): def main(): - import getopt - - opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['help', 'command=']) - - if not args: - print(_usage) + import argparse + + parser = argparse.ArgumentParser(prog="pdb", + description=_usage, + formatter_class=argparse.RawDescriptionHelpFormatter, + allow_abbrev=False) + + parser.add_argument('-c', '--command', action='append', default=[], metavar='command') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-m', metavar='module') + group.add_argument('pyfile', nargs='?') + parser.add_argument('args', nargs="*") + + if len(sys.argv) == 1: + # If no arguments were given (python -m pdb), print the whole help message. + # Without this check, argparse would only complain about missing required arguments. + parser.print_help() sys.exit(2) - if any(opt in ['-h', '--help'] for opt, optarg in opts): - print(_usage) - sys.exit() - - commands = [optarg for opt, optarg in opts if opt in ['-c', '--command']] + opts = parser.parse_args() - module_indicated = any(opt in ['-m'] for opt, optarg in opts) - cls = _ModuleTarget if module_indicated else _ScriptTarget - target = cls(args[0]) + if opts.m: + file = opts.m + target = _ModuleTarget(file) + else: + file = opts.pyfile + target = _ScriptTarget(file) target.check() - sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list + sys.argv[:] = [file] + opts.args # Hide "pdb.py" and pdb options from argument list # Note on saving/restoring sys.argv: it's a good idea when sys.argv was # modified by the script being debugged. It's a bad idea when it was # changed by the user from the command line. There is a "restart" command # which allows explicit specification of command line arguments. pdb = Pdb() - pdb.rcLines.extend(commands) + pdb.rcLines.extend(opts.command) while True: try: pdb._run(target) - if pdb._user_requested_quit: - break - print("The program finished and will be restarted") except Restart: print("Restarting", target, "with arguments:") print("\t" + " ".join(sys.argv[1:])) @@ -1917,17 +2231,16 @@ def main(): # In most cases SystemExit does not warrant a post-mortem session. print("The program exited via sys.exit(). Exit status:", end=' ') print(e) - except SyntaxError: - traceback.print_exc() - sys.exit(1) except BaseException as e: traceback.print_exc() print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") - t = e.__traceback__ - pdb.interaction(None, t) + pdb.interaction(None, e) print("Post mortem debugger finished. The " + target + " will be restarted") + if pdb._user_requested_quit: + break + print("The program finished and will be restarted") # When invoked as main program, invoke the debugger on a script diff --git a/Lib/pickle.py b/Lib/pickle.py index fe86f80f51d3b9..4f5ad5b71e8899 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -396,6 +396,8 @@ def decode_long(data): return int.from_bytes(data, byteorder='little', signed=True) +_NoValue = object() + # Pickling machinery class _Pickler: @@ -542,8 +544,8 @@ def save(self, obj, save_persistent_id=True): return rv = NotImplemented - reduce = getattr(self, "reducer_override", None) - if reduce is not None: + reduce = getattr(self, "reducer_override", _NoValue) + if reduce is not _NoValue: rv = reduce(obj) if rv is NotImplemented: @@ -556,8 +558,8 @@ def save(self, obj, save_persistent_id=True): # Check private dispatch table if any, or else # copyreg.dispatch_table - reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) - if reduce is not None: + reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue) + if reduce is not _NoValue: rv = reduce(obj) else: # Check for a class with a custom metaclass; treat as regular @@ -567,12 +569,12 @@ def save(self, obj, save_persistent_id=True): return # Check for a __reduce_ex__ method, fall back to __reduce__ - reduce = getattr(obj, "__reduce_ex__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce_ex__", _NoValue) + if reduce is not _NoValue: rv = reduce(self.proto) else: - reduce = getattr(obj, "__reduce__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce__", _NoValue) + if reduce is not _NoValue: rv = reduce() else: raise PicklingError("Can't pickle %r object: %r" % @@ -1705,8 +1707,8 @@ def load_build(self): stack = self.stack state = stack.pop() inst = stack[-1] - setstate = getattr(inst, "__setstate__", None) - if setstate is not None: + setstate = getattr(inst, "__setstate__", _NoValue) + if setstate is not _NoValue: setstate(state) return slotstate = None diff --git a/Lib/pipes.py b/Lib/pipes.py deleted file mode 100644 index 61d63b48d3e4c5..00000000000000 --- a/Lib/pipes.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Conversion pipeline templates. - -The problem: ------------- - -Suppose you have some data that you want to convert to another format, -such as from GIF image format to PPM image format. Maybe the -conversion involves several steps (e.g. piping it through compress or -uuencode). Some of the conversion steps may require that their input -is a disk file, others may be able to read standard input; similar for -their output. The input to the entire conversion may also be read -from a disk file or from an open file, and similar for its output. - -The module lets you construct a pipeline template by sticking one or -more conversion steps together. It will take care of creating and -removing temporary files if they are necessary to hold intermediate -data. You can then use the template to do conversions from many -different sources to many different destinations. The temporary -file names used are different each time the template is used. - -The templates are objects so you can create templates for many -different conversion steps and store them in a dictionary, for -instance. - - -Directions: ------------ - -To create a template: - t = Template() - -To add a conversion step to a template: - t.append(command, kind) -where kind is a string of two characters: the first is '-' if the -command reads its standard input or 'f' if it requires a file; the -second likewise for the output. The command must be valid /bin/sh -syntax. If input or output files are required, they are passed as -$IN and $OUT; otherwise, it must be possible to use the command in -a pipeline. - -To add a conversion step at the beginning: - t.prepend(command, kind) - -To convert a file to another file using a template: - sts = t.copy(infile, outfile) -If infile or outfile are the empty string, standard input is read or -standard output is written, respectively. The return value is the -exit status of the conversion pipeline. - -To open a file for reading or writing through a conversion pipeline: - fp = t.open(file, mode) -where mode is 'r' to read the file, or 'w' to write it -- just like -for the built-in function open() or for os.popen(). - -To create a new template object initialized to a given one: - t2 = t.clone() -""" # ' - - -import re -import os -import tempfile -import warnings -# we import the quote function rather than the module for backward compat -# (quote used to be an undocumented but used function in pipes) -from shlex import quote - -warnings._deprecated(__name__, remove=(3, 13)) - -__all__ = ["Template"] - -# Conversion step kinds - -FILEIN_FILEOUT = 'ff' # Must read & write real files -STDIN_FILEOUT = '-f' # Must write a real file -FILEIN_STDOUT = 'f-' # Must read a real file -STDIN_STDOUT = '--' # Normal pipeline element -SOURCE = '.-' # Must be first, writes stdout -SINK = '-.' # Must be last, reads stdin - -stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \ - SOURCE, SINK] - - -class Template: - """Class representing a pipeline template.""" - - def __init__(self): - """Template() returns a fresh pipeline template.""" - self.debugging = 0 - self.reset() - - def __repr__(self): - """t.__repr__() implements repr(t).""" - return '

Cette - recipie - déjeuner. + recette + sera sûrement un très bon repas.