From 5af3f627ba688c34a2bfac33dde5a65d9bc5a370 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 18 May 2024 01:52:52 -0500 Subject: [PATCH 1/5] [python-package] drop Python 3.6 support, add Python 3.12 --- .ci/conda-envs/ci-core-py38.txt | 51 +++++++++++++++++++++++++++ .ci/test-python-oldest.sh | 1 - .ci/test.sh | 2 ++ .ci/test_windows.ps1 | 2 ++ .github/workflows/cuda.yml | 6 ++-- .github/workflows/linkchecker.yml | 2 +- .github/workflows/python_package.yml | 16 ++++----- .github/workflows/static_analysis.yml | 2 +- .vsts-ci.yml | 22 ++++++------ python-package/pyproject.toml | 4 +-- 10 files changed, 81 insertions(+), 27 deletions(-) create mode 100644 .ci/conda-envs/ci-core-py38.txt diff --git a/.ci/conda-envs/ci-core-py38.txt b/.ci/conda-envs/ci-core-py38.txt new file mode 100644 index 000000000000..7fcd986b40f8 --- /dev/null +++ b/.ci/conda-envs/ci-core-py38.txt @@ -0,0 +1,51 @@ +# [description] +# +# Similar to ci-core.txt, but specific to Python 3.8. +# +# Unlike ci-core.txt, this includes a Python version and uses +# `=` and `<=` pins to make solves faster and prevent against +# issues like https://github.com/microsoft/LightGBM/pull/6370. +# +# [usage] +# +# conda create \ +# --name test-env \ +# --file ./.ci/conda-envs/ci-core-py38.txt +# + +# python +python=3.8.* + +# direct imports +cffi=1.15.* +dask=2023.5.* +distributed=2023.5.* +joblib=1.4.* +matplotlib-base=3.7.* +numpy=1.24.* +pandas=1.5.* +pyarrow-core=16.1.* +python-graphviz=0.20.* +scikit-learn=1.3.* +scipy=1.10.* + +# testing-only dependencies +cloudpickle=3.0.* +pluggy=1.5.* +psutil=5.9.8 +pytest=8.2.* + +# other recursive dependencies, just +# pinned here to help speed up solves +bokeh=3.1.* +fsspec=2024.5.* +msgpack-python=1.0.* +pluggy=1.5.* +pytz=2024.1 +setuptools=69.5.* +snappy=1.2.* +tomli=2.0.* +tornado=6.4.* +wheel=0.43.* +zict=3.0.* +zipp=3.17.* diff --git a/.ci/test-python-oldest.sh b/.ci/test-python-oldest.sh index 7f1c586e1f22..ff3f15f79429 100644 --- a/.ci/test-python-oldest.sh +++ b/.ci/test-python-oldest.sh @@ -10,7 +10,6 @@ set -e -E -u -o pipefail echo "installing lightgbm's dependencies" pip install \ 'cffi==1.15.1' \ - 'dataclasses' \ 'numpy==1.16.6' \ 'pandas==0.24.0' \ 'pyarrow==6.0.1' \ diff --git a/.ci/test.sh b/.ci/test.sh index 1b8a31340130..7de477a5571d 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -130,6 +130,8 @@ fi if [[ $PYTHON_VERSION == "3.7" ]]; then CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt" +elif [[ $PYTHON_VERSION == "3.8" ]]; then + CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt" else CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt" fi diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index 12f90f71cd05..bb8353e159b3 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -53,6 +53,8 @@ conda update -q -y conda if ($env:PYTHON_VERSION -eq "3.7") { $env:CONDA_REQUIREMENT_FILE = "$env:BUILD_SOURCESDIRECTORY/.ci/conda-envs/ci-core-py37.txt" +} elseif ($env:PYTHON_VERSION -eq "3.8") { + $env:CONDA_REQUIREMENT_FILE = "$env:BUILD_SOURCESDIRECTORY/.ci/conda-envs/ci-core-py38.txt" } else { $env:CONDA_REQUIREMENT_FILE = "$env:BUILD_SOURCESDIRECTORY/.ci/conda-envs/ci-core.txt" } diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index b7f825b245b8..3fe60237a3bc 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -30,17 +30,17 @@ jobs: include: - method: wheel compiler: gcc - python_version: "3.11" + python_version: "3.10" cuda_version: "11.8.0" task: cuda - method: source compiler: gcc - python_version: "3.9" + python_version: "3.12" cuda_version: "12.2.0" task: cuda - method: pip compiler: clang - python_version: "3.10" + python_version: "3.11" cuda_version: "11.8.0" task: cuda steps: diff --git a/.github/workflows/linkchecker.yml b/.github/workflows/linkchecker.yml index fe0176ac6b9d..8bea60c02bda 100644 --- a/.github/workflows/linkchecker.yml +++ b/.github/workflows/linkchecker.yml @@ -11,7 +11,7 @@ env: CONDA_ENV: test-env GITHUB_ACTIONS: 'true' OS_NAME: 'linux' - PYTHON_VERSION: '3.11' + PYTHON_VERSION: '3.12' TASK: 'check-links' jobs: diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 8b2ed1dcc5c1..b755668ba3fe 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -29,30 +29,30 @@ jobs: include: - os: macos-13 task: regular - python_version: '3.9' + python_version: '3.10' - os: macos-13 task: sdist - python_version: '3.10' + python_version: '3.11' - os: macos-13 task: bdist - python_version: '3.7' + python_version: '3.8' - os: macos-13 task: if-else - python_version: '3.9' + python_version: '3.10' # We're currently skipping MPI jobs on macOS, see https://github.com/microsoft/LightGBM/pull/6425 # for further details. # - os: macos-13 # task: mpi # method: source - # python_version: '3.10' + # python_version: '3.11' # - os: macos-13 # task: mpi # method: pip - # python_version: '3.11' + # python_version: '3.12' # - os: macos-13 # task: mpi # method: wheel - # python_version: '3.8' + # python_version: '3.9' steps: - name: Checkout repository uses: actions/checkout@v3 @@ -102,7 +102,7 @@ jobs: --rm \ -v $(pwd):/opt/lgb-build \ -w /opt/lgb-build \ - python:3.6 \ + python:3.7 \ /bin/bash ./.ci/test-python-oldest.sh all-python-package-jobs-successful: if: always() diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index eb84e41d4a83..5eee2e23367a 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -21,7 +21,7 @@ env: CONDA_ENV: test-env GITHUB_ACTIONS: 'true' OS_NAME: 'linux' - PYTHON_VERSION: '3.11' + PYTHON_VERSION: '3.12' jobs: test: diff --git a/.vsts-ci.yml b/.vsts-ci.yml index b9cba8d78671..e33e20820714 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -10,7 +10,7 @@ pr: - release/* variables: AZURE: 'true' - PYTHON_VERSION: '3.11' + PYTHON_VERSION: '3.12' CONDA_ENV: test-env runCodesignValidationInjection: false skipComponentGovernanceDetection: true @@ -61,19 +61,19 @@ jobs: matrix: regular: TASK: regular - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' sdist: TASK: sdist - PYTHON_VERSION: '3.7' + PYTHON_VERSION: '3.8' bdist: TASK: bdist - PYTHON_VERSION: '3.8' + PYTHON_VERSION: '3.9' inference: TASK: if-else mpi_source: TASK: mpi METHOD: source - PYTHON_VERSION: '3.8' + PYTHON_VERSION: '3.9' gpu_source: TASK: gpu METHOD: source @@ -127,7 +127,7 @@ jobs: TASK: sdist bdist: TASK: bdist - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' inference: TASK: if-else mpi_source: @@ -136,23 +136,23 @@ jobs: mpi_pip: TASK: mpi METHOD: pip - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' mpi_wheel: TASK: mpi METHOD: wheel - PYTHON_VERSION: '3.8' + PYTHON_VERSION: '3.9' gpu_source: TASK: gpu METHOD: source - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' gpu_pip: TASK: gpu METHOD: pip - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' gpu_wheel: TASK: gpu METHOD: wheel - PYTHON_VERSION: '3.8' + PYTHON_VERSION: '3.9' cpp_tests: TASK: cpp-tests METHOD: with-sanitizers diff --git a/python-package/pyproject.toml b/python-package/pyproject.toml index 387fbd2e407a..b7bff79edfc8 100644 --- a/python-package/pyproject.toml +++ b/python-package/pyproject.toml @@ -15,10 +15,10 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Artificial Intelligence" ] dependencies = [ - "dataclasses ; python_version < '3.7'", "numpy", "scipy" ] @@ -29,7 +29,7 @@ maintainers = [ ] name = "lightgbm" readme = "README.rst" -requires-python = ">=3.6" +requires-python = ">=3.7" version = "4.3.0.99" [project.optional-dependencies] From 42b4d98704c5a0802dc4b7a20bc94d315b15e3b3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 18 May 2024 18:35:26 -0500 Subject: [PATCH 2/5] update versions to get wheels --- .ci/test-python-oldest.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.ci/test-python-oldest.sh b/.ci/test-python-oldest.sh index ff3f15f79429..c6de079351e3 100644 --- a/.ci/test-python-oldest.sh +++ b/.ci/test-python-oldest.sh @@ -3,18 +3,20 @@ set -e -E -u -o pipefail # oldest versions of dependencies published after -# minimum supported Python version's first release +# minimum supported Python version's first release, +# for which there are wheels compatible with the +# python:{version} image # # see https://devguide.python.org/versions/ # echo "installing lightgbm's dependencies" pip install \ 'cffi==1.15.1' \ - 'numpy==1.16.6' \ - 'pandas==0.24.0' \ + 'numpy==1.19.0' \ + 'pandas==1.1.3' \ 'pyarrow==6.0.1' \ - 'scikit-learn==0.18.2' \ - 'scipy==0.19.0' \ + 'scikit-learn==0.24.0' \ + 'scipy==1.6.0' \ || exit 1 echo "done installing lightgbm's dependencies" From 83336a815f4624cc1d541625de6466d0680120cb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 23 May 2024 21:12:54 -0500 Subject: [PATCH 3/5] get more logs --- .ci/check_python_dists.sh | 3 +++ .ci/test.sh | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.ci/check_python_dists.sh b/.ci/check_python_dists.sh index 59547722508b..debae9de0e45 100644 --- a/.ci/check_python_dists.sh +++ b/.ci/check_python_dists.sh @@ -12,6 +12,7 @@ echo "checking Python package distributions in '${DIST_DIR}'" pip install \ -qq \ + auditwheel \ check-wheel-contents \ twine || exit 1 @@ -21,6 +22,8 @@ twine check --strict ${DIST_DIR}/* || exit 1 if { test "${TASK}" = "bdist" || test "${METHOD}" = "wheel"; }; then echo "check-wheel-contents..." check-wheel-contents ${DIST_DIR}/*.whl || exit 1 + echo "auditwheel show ..." + auditwheel show ${DIST_DIR}/*.whl || exit 1 fi PY_MINOR_VER=$(python -c "import sys; print(sys.version_info.minor)") diff --git a/.ci/test.sh b/.ci/test.sh index 1d59288d5ecb..170ec9980388 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -2,6 +2,8 @@ set -e -E -o -u pipefail +export LD_DEBUG=libs + # defaults IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} METHOD=${METHOD:-""} From 976833a869e4c3da3dac630158bca90a3cd11d03 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 23 May 2024 21:46:39 -0500 Subject: [PATCH 4/5] more specific logs --- .ci/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index 170ec9980388..3c8ccd4304c9 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -2,8 +2,6 @@ set -e -E -o -u pipefail -export LD_DEBUG=libs - # defaults IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} METHOD=${METHOD:-""} @@ -156,6 +154,8 @@ if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then for LIBOMP_ALIAS in libgomp.dylib libiomp5.dylib libomp.dylib; do sudo ln -sf "$(brew --cellar libomp)"/*/lib/libomp.dylib $CONDA_PREFIX/lib/$LIBOMP_ALIAS || exit 1; done fi +export LD_DEBUG=libs + if [[ $TASK == "sdist" ]]; then cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit 1 sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1 @@ -247,13 +247,13 @@ elif [[ $TASK == "cuda" ]]; then --config-settings=cmake.define.USE_CUDA=ON \ $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz \ || exit 1 - pytest $BUILD_DIRECTORY/tests/python_package_test || exit 1 + pytest $BUILD_DIRECTORY/tests/python_package_test/test_basic.py || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --cuda || exit 1 sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit 1 pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit 1 - pytest $BUILD_DIRECTORY/tests || exit 1 + pytest $BUILD_DIRECTORY/tests/python_package_test/test_basic.py || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then cmake -B build -S . -DUSE_CUDA=ON From 72d8821406584fffce18847682f1806d379c0a2e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 23 May 2024 22:18:48 -0500 Subject: [PATCH 5/5] get verbose output from cmake --- python-package/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/pyproject.toml b/python-package/pyproject.toml index b7bff79edfc8..5b3232fa731e 100644 --- a/python-package/pyproject.toml +++ b/python-package/pyproject.toml @@ -69,7 +69,7 @@ ninja.make-fallback = true cmake.args = [ "-D__BUILD_FOR_PYTHON:BOOL=ON" ] -cmake.verbose = false +cmake.verbose = true cmake.build-type = "Release" cmake.targets = ["_lightgbm"] # stripping binaries should be turned back on once this is fixed: