Merge branch 'master' into ENH_worker_id

joblib · Apr 9, 2024 · aaf6100 · aaf6100
2 parents aac9234 + 2c21e44
commit aaf6100
Show file tree

Hide file tree

Showing 69 changed files with 3,730 additions and 4,112 deletions.
diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml
@@ -1,71 +1,70 @@
 jobs:
+- job: linting
+  displayName: Linting
+  pool:
+    vmImage: ubuntu-latest
+  steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.9'
+    - bash: |
+        # Include pytest compatibility with mypy
+        pip install black==22.3
+      displayName: Install linters
+    - bash: |
+        black --check --diff .
+      displayName: Run black
+
 - job: 'test_loky'
   strategy:
     matrix:
 
+      windows-py311:
+        imageName: windows-latest
+        python.version: "3.11"
+        tox.env: py311
       windows-py38:
-        imageName: "vs2017-win2016"
+        imageName: windows-latest
         python.version: "3.8"
         tox.env: py38
-      windows-py35:
-        imageName: "vs2017-win2016"
-        python.version: "3.5"
-        tox.env: py35
-      windows-py27:
-        imageName: "vs2017-win2016"
-        python.version: "2.7"
-        tox.env: py27
 
+      macos-py311:
+        imageName: "macos-latest"
+        python.version: "3.11"
+        tox.env: py311
       macos-py38:
-        imageName: "macos-10.14"
+        imageName: "macos-latest"
         python.version: "3.8"
         tox.env: py38
-      macos-py35:
-        imageName: "macos-10.14"
-        python.version: "3.5"
-        tox.env: py35
-      macos-py27-high-memory:
-        imageName: "macos-10.14"
-        python.version: "2.7"
-        tox.env: py27
-        RUN_MEMORY: "true"
 
       linux-pypy3:
-        imageName: "ubuntu-16.04"
+        imageName: "ubuntu-latest"
         python.version: "pypy3"
         tox.env: pypy3
         LOKY_MAX_CPU_COUNT: "2"
 
-      linux-python-nightly:
-        imageName: "ubuntu-16.04"
+      linux-py311:
+        imageName: "ubuntu-latest"
+        python.version: "3.11"
+        tox.env: py311
+      linux-py39-joblib-tests:
+        imageName: "ubuntu-latest"
         python.version: "3.9"
-        tox.env: py39
-      linux-py38-joblib-tests:
-        imageName: "ubuntu-16.04"
-        python.version: "3.8"
-        tox.env: "py38"
+        tox.env: "py39"
         joblib.tests: "true"
-      linux-py38-high-memory:
-        imageName: "ubuntu-16.04"
+      linux-python-py39-high-memory:
+        imageName: "ubuntu-latest"
+        python.version: "3.9"
+        tox.env: py39
+        RUN_MEMORY: "true"
+      linux-py38:
+        imageName: "ubuntu-latest"
         python.version: "3.8"
         tox.env: py38
-        RUN_MEMORY: "true"
       linux-py37:
-        imageName: "ubuntu-16.04"
+        imageName: "ubuntu-latest"
         python.version: "3.7"
         tox.env: py37
-      linux-py36:
-        imageName: "ubuntu-16.04"
-        python.version: "3.6"
-        tox.env: py36
-      linux-py35:
-        imageName: "ubuntu-16.04"
-        python.version: "3.5"
-        tox.env: py35
-      linux-py27:
-        imageName: "ubuntu-16.04"
-        python.version: "2.7"
-        tox.env: py27
 
   pool:
     vmImage: $(imageName)
@@ -77,15 +76,18 @@ jobs:
       inputs:
         versionSpec: '$(python.version)'
       displayName: 'Use Python $(python.version)'
-      condition: ne(variables['python.version'], '3.9')
+      condition: ne(variables['python.version'], 'pypy3')
     - bash: |
-        sudo add-apt-repository ppa:deadsnakes/nightly
-        sudo apt update
-        sudo apt install python3.9 python3.9-venv python3.9-dev
-        python3.9 -m venv nightly-venv
-        echo "##vso[task.prependpath]nightly-venv/bin"
-      displayName: Install Python 3.9 from ppa:deadsnakes/nightly
-      condition: eq(variables['python.version'], '3.9')
+        wget -q https://downloads.python.org/pypy/pypy3.7-v7.3.7-linux64.tar.bz2 -O pypy.tar.bz2
+        mkdir -p pypy3
+        (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2)
+        mv pypy3 /opt/pypy37
+        /opt/pypy37/bin/pypy3 -m ensurepip
+        /opt/pypy37/bin/pypy3 -m pip install --upgrade pip wheel
+        /opt/pypy37/bin/pypy3 -m venv pypy-venv
+        echo "##vso[task.prependpath]pypy-venv/bin"
+      displayName: 'Install PyPy 3.7'
+      condition: eq(variables['python.version'], 'pypy3')
     # azure-pipelines unpredictably switches between Git\bin\bash and
     # Git\usr\bin\bash when running a bash script inside Windows environments.
     # The latter may use wrong bash commands, resulting in errors when codecov

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,11 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    -   id: black
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -1,5 +1,22 @@
-requirements_file: docs/requirements.txt
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+version: 2
 
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
-   setup_py_install: true
-   version: 3.8
+  install:
+    - method: pip
+      path: .
+    - requirements: docs/requirements.txt
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,8 +1,72 @@
-### 3.0.0 - XXXX-YY-ZZ
+### 3.5.0 - in development
+
+
+### 3.4.1 - 2023-06-29
+
+- Fix compatibility with python3.7, which does not define
+  a `_MAX_WINDOWS_WORKERS` constant. (#408)
+
+### 3.4.0 - 2023-04-14
+
+- Fix exception `__cause__` not being propagated with
+  `tblib.pickling_support.install()`. (#255).
+
+- Fix handling of CPU affinity  by using `psutil`'s `cpu_affinity` on platforms
+  that do not implement `os.sched_getaffinity`, such as PyPy. (#381).
+
+- Make the executor's gc process more thread-safe, in particular for PyPy,
+  where the gc calls can be run in any thread. (#384).
+
+- Fix crash when using `max_workers > 61` on Windows. Loky will no longer
+  attempt to use more than 61 workers on that platform (or 60 depending on the
+  Python version). (#390).
+
+- Fix loky compat with python 3.11 for nested calls. (#394).
+
+- Adapt the cooldown strategy when shutingdown an executor with full
+  `call_queue`. This should accelerate the time taken to shutdown
+  in general, in particular on overloaded machines. (#399).
+
+### 3.3.0 - 2022-09-15
+
+- Fix worker management logic in `get_reusable_executor` to ensure
+  the number of started worker process actually correspond to `max_workers`
+  when existing process concurrently time out (#370).
+
+### 3.2.0 - 2022-09-14
+
+- Fix leaked processes and deadlock when the Python interpreter exits
+  after a using nested calls to `get_reusable_executor` (#363).
+
+- Fix an exception in the SemLock finalizer when the semaphore has been
+  concurrently unlinked (#366).
+
+### 3.1.0 - 2022-02-22
+
+- Fix loky.cpu_count() to properly detect the number of allowed CPUs based on
+  the /sys/fs/cgroup/cpu.max file on newest Linux versions with cgroup v2.
+  Fall-back to the /sys/fs/cgroup/cpu/cpu.cfs_quota_us file to keep on
+  supporting Linux versions that use cgroup v1 (#355 and #358).
+
+- Fix an exception that could be raised in an auxiliary thread when
+  garbage collecting an executor instance when shutting down the
+  the Python interpreter (#311).
+
+- Make `shutdown(kill_workers=True)` consistently use the SIGKILL
+  signal on POSIX. Previously a mix of SIGKILL and SIGTERM was issued
+  and could deadlock the shutdown process (#348 and #357).
+
+- Big code clean-up to drop support for older Python versions.
+  Python 3.7 or later is now required. (#304)
+
+### 3.0.0 - 2021-09-10
 
 - Avoid a NameError when calling the `exit` builtin on Windows when
   loky is executed as part of a frozen Python binary. (#290)
 
+- Make it possible to automatically trace workers when profiling with
+  VizTracer (#299).
+
 ### 2.9.0 - 2020-10-02
 
 - Fix a side-effect bug in the registration of custom reducers the loky

diff --git a/CITATION.cff b/CITATION.cff
@@ -0,0 +1,20 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+- family-names: "Moreau"
+  given-names: "Thomas"
+  orcid: "https://orcid.org/0000-0002-1523-3419"
+- family-names: "Glaser"
+  given-names: "Pierre"
+  orcid: "https://orcid.org/0000-0003-4146-973X"
+- family-names: "Yurchak"
+  given-names: "Roman"
+  orcid: "https://orcid.org/0000-0002-2565-4444"
+- family-names: "Grisel"
+  given-names: "Olivier"
+  orcid: "https://orcid.org/0000-0002-8226-4012"
+title: "Loky"
+version: latest
+doi: 10.5281/zenodo.5770406
+date-released: 2017-06-20
+url: "https://github.com/joblib/loky"
diff --git a/README.md b/README.md
@@ -8,6 +8,7 @@ alt="Loky logo" width=96/></a>
 [![Build Status](https://dev.azure.com/joblib/loky/_apis/build/status/joblib.loky?branchName=master)](https://dev.azure.com/joblib/loky/_build/latest?definitionId=2&branchName=master)
 [![Documentation Status](https://readthedocs.org/projects/loky/badge/?version=latest)](https://loky.readthedocs.io/en/latest/?badge=latest)
 [![codecov](https://codecov.io/gh/joblib/loky/branch/master/graph/badge.svg)](https://codecov.io/gh/joblib/loky)
+[![DOI](https://zenodo.org/badge/48578152.svg)](https://zenodo.org/badge/latestdoi/48578152)
 
 
 ### Goal
@@ -45,7 +46,7 @@ cross-version implementation of the `ProcessPoolExecutor` class of
     processes. This library intends to fix those possible deadlocks and
     send back meaningful errors. Note that the implementation of
     `concurrent.futures.ProcessPoolExecutor` that comes with Python 3.7+
-    is as robust as the executor from loky but the later also works for
+    is as robust as the executor from loky but the latter also works for
     older versions of Python.
 
 
@@ -80,7 +81,7 @@ from loky import get_reusable_executor
 
 def say_hello(k):
     pid = os.getpid()
-    print("Hello from {} with arg {}".format(pid, k))
+    print(f"Hello from {pid} with arg {k}")
     sleep(.01)
     return pid
 

diff --git a/README.rst b/README.rst
@@ -59,7 +59,7 @@ The basic usage of :mod:`loky` relies on the :func:`~loky.get_reusable_executor`
 
     def say_hello(k):
         pid = os.getpid()
-        print("Hello from {} with arg {}".format(pid, k))
+        print(f"Hello from {pid} with arg {k}")
         sleep(.01)
         return pid
 

diff --git a/continuous_integration/install_coverage_subprocess_pth.py b/continuous_integration/install_coverage_subprocess_pth.py
@@ -5,12 +5,12 @@
 import os.path as op
 from distutils.sysconfig import get_python_lib
 
-FILE_CONTENT = u"""\
+FILE_CONTENT = """\
 import coverage; coverage.process_startup()
 """
 
-filename = op.join(get_python_lib(), 'coverage_subprocess.pth')
-with open(filename, 'wb') as f:
-    f.write(FILE_CONTENT.encode('ascii'))
+filename = op.join(get_python_lib(), "coverage_subprocess.pth")
+with open(filename, mode="w") as f:
+    f.write(FILE_CONTENT)
 
-print('Installed subprocess coverage support: %s' % filename)
+print(f"Installed subprocess coverage support: {filename}")
diff --git a/continuous_integration/runtests.sh b/continuous_integration/runtests.sh
@@ -15,7 +15,7 @@ if [ "$JOBLIB_TESTS" = "true" ]; then
     which python
     git clone https://github.com/joblib/joblib.git src_joblib
     cd src_joblib
-    pip install pytest
+    pip install "pytest<7.0"  # Need to update remove occurrences of pytest.warns(None)
     pip install threadpoolctl  # required by some joblib tests
 
     pip install -e .
@@ -27,7 +27,7 @@ else
     # Make sure that we have the python docker image cached locally to avoid
     # a timeout in a test that needs it.
     if [ "$(which docker)" != "" ] && [ "$(uname)" = "Linux" ]; then
-        docker pull python:3.6
+        docker pull python:3.10
     fi
 
     # Run the tests and collect trace coverage data both in the subprocesses

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,7 +1,6 @@
 numpy
 psutil
 cython
-faulthandler ; python_version < '3.3'
 coverage
 pytest
 pytest-coverage

diff --git a/docs/API.rst b/docs/API.rst
@@ -5,6 +5,10 @@ API Reference
 .. automodule:: loky
     :members: get_reusable_executor
 
+.. autoclass:: ProcessPoolExecutor
+    :members:
+    :class-doc-from: init
+
 
 Task & results serialization
 ----------------------------
@@ -55,8 +59,8 @@ not happening.
 
 .. rubric:: Footnotes
 
-.. [#periodically_fn] every 1 second. This constant is define in :code:`loky.process_executor._MEMORY_LEAK_CHECK_DELAY`
-.. [#psutil_unusual_fn] an increase of 100MB compared to a reference, which is defined as the residual memory usage of the worker after it completed its first task
+.. [#periodically_fn] every 1 second. This constant is defined in :code:`loky.process_executor._MEMORY_LEAK_CHECK_DELAY`
+.. [#psutil_unusual_fn] an increase of 300MB compared to a reference, which is defined as the residual memory usage of the worker after it completed its first task. This constant is defined in :code:`loky.process_executor._MAX_MEMORY_LEAK_SIZE`