From fcafd111af6c323d9da569370ad6f66e99e46c61 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Tue, 14 Jan 2025 18:24:50 +0800 Subject: [PATCH 01/42] Build R docs with pkgdown. install dir. Restore. system dependencies. Update ubuntu version. work on script. --- .github/workflows/r_tests.yml | 22 ++++++++++ R-package/.Rbuildignore | 3 ++ R-package/.gitignore | 1 + R-package/README.md | 12 ++---- R-package/pkgdown/_pkgdown.yml | 4 ++ R-package/vignettes/xgboost_introduction.Rmd | 15 ++++--- doc/R-package/index.rst | 9 ++++ doc/conf.py | 43 +++++++++++++++++--- ops/pipeline/build-r-docs-impl.sh | 24 +++++++++++ ops/pipeline/build-r-docs.sh | 19 +++++++++ 10 files changed, 132 insertions(+), 20 deletions(-) create mode 100644 R-package/.gitignore create mode 100644 R-package/pkgdown/_pkgdown.yml create mode 100644 ops/pipeline/build-r-docs-impl.sh create mode 100644 ops/pipeline/build-r-docs.sh diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml index 43ad372a1e84..57131b96b96d 100644 --- a/.github/workflows/r_tests.yml +++ b/.github/workflows/r_tests.yml @@ -101,3 +101,25 @@ jobs: if: steps.changes.outputs.r_package == 'true' run: | python3 ops/script/test_r_package.py --r=/usr/bin/R --task=doc + + build-r-docs: + name: Build docs for the R package + runs-on: + - runs-on=${{ github.run_id }} + - runner=linux-amd64-cpu + - tag=r-tests-build-jvm-docs + steps: + # Restart Docker daemon so that it recognizes the ephemeral disks + - run: sudo systemctl restart docker + - uses: actions/checkout@v4 + with: + submodules: "true" + - name: Log into Docker registry (AWS ECR) + run: bash ops/pipeline/login-docker-registry.sh + - run: bash ops/pipeline/build-r-docs.sh + - name: Upload R doc + run: | + python3 ops/pipeline/manage-artifacts.py upload \ + --s3-bucket xgboost-docs \ + --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \ + R-package/r-docs-${{ env.BRANCH_NAME }}.tar.bz2 diff --git a/R-package/.Rbuildignore b/R-package/.Rbuildignore index b1932e324589..ca9c76e2411a 100644 --- a/R-package/.Rbuildignore +++ b/R-package/.Rbuildignore @@ -6,3 +6,6 @@ README.md ^doc$ ^Meta$ +^_pkgdown\.yml$ +^docs$ +^pkgdown$ diff --git a/R-package/.gitignore b/R-package/.gitignore new file mode 100644 index 000000000000..d8f8d46921aa --- /dev/null +++ b/R-package/.gitignore @@ -0,0 +1 @@ +docs diff --git a/R-package/README.md b/R-package/README.md index f68b1954be7d..50f773755e6f 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -3,11 +3,11 @@ XGBoost R Package for Scalable GBM [![CRAN Status Badge](http://www.r-pkg.org/badges/version/xgboost)](https://cran.r-project.org/web/packages/xgboost) [![CRAN Downloads](http://cranlogs.r-pkg.org/badges/xgboost)](https://cran.rstudio.com/web/packages/xgboost/index.html) -[![Documentation Status](https://readthedocs.org/projects/xgboost/badge/?version=latest)](http://xgboost.readthedocs.org/en/latest/R-package/index.html) +[![Documentation Status](https://readthedocs.org/projects/xgboost/badge/?version=latest)](https://xgboost.readthedocs.org/en/latest/R-package/index.html) Resources --------- -* [XGBoost R Package Online Documentation](http://xgboost.readthedocs.org/en/latest/R-package/index.html) +* [XGBoost R Package Online Documentation](https://xgboost.readthedocs.org/en/latest/R-package/index.html) - Check this out for detailed documents, examples and tutorials. Installation @@ -19,13 +19,7 @@ We are [on CRAN](https://cran.r-project.org/web/packages/xgboost/index.html) now install.packages('xgboost') ``` -For more detailed installation instructions, please see [here](http://xgboost.readthedocs.org/en/latest/build.html#r-package-installation). - -Examples --------- - -* Please visit [walk through example](demo). -* See also the [example scripts](../demo/kaggle-higgs) for Kaggle Higgs Challenge, including [speedtest script](../demo/kaggle-higgs/speedtest.R) on this dataset and the one related to [Otto challenge](../demo/kaggle-otto), including a [RMarkdown documentation](../demo/kaggle-otto/understandingXGBoostModel.Rmd). +For more detailed installation instructions, please see [here](https://xgboost.readthedocs.io/en/stable/install.html). Development ----------- diff --git a/R-package/pkgdown/_pkgdown.yml b/R-package/pkgdown/_pkgdown.yml new file mode 100644 index 000000000000..3a3c60f97169 --- /dev/null +++ b/R-package/pkgdown/_pkgdown.yml @@ -0,0 +1,4 @@ +url: https://github.com/dmlc/xgboost + +template: + bootstrap: 5 diff --git a/R-package/vignettes/xgboost_introduction.Rmd b/R-package/vignettes/xgboost_introduction.Rmd index 8d3573f11f5f..6a08dd29eaec 100644 --- a/R-package/vignettes/xgboost_introduction.Rmd +++ b/R-package/vignettes/xgboost_introduction.Rmd @@ -12,7 +12,10 @@ output: toc_float: true --- -# Introduction +XGBoost for R introduction +========================== + +## Introduction **XGBoost** is an optimized distributed gradient boosting library designed to be highly **efficient**, **flexible** and **portable**. It implements machine learning algorithms under the [Gradient Boosting](https://en.wikipedia.org/wiki/Gradient_boosting) framework. XGBoost provides a parallel tree boosting (also known as GBDT, GBM) that solve many data science problems in a fast and accurate way. The same code runs on major distributed environment (Hadoop, SGE, MPI) and can solve problems beyond billions of examples. @@ -22,7 +25,7 @@ For more details about XGBoost's features and usage, see the [online documentati This short vignette outlines the basic usage of the R interface for XGBoost, assuming the reader has some familiarity with the underlying concepts behind statistical modeling with gradient-boosted decision trees. -# Building a predictive model +## Building a predictive model At its core, XGBoost consists of a C++ library which offers bindings for different programming languages, including R. The R package for XGBoost provides an idiomatic interface similar to those of other statistical modeling packages using and x/y design, as well as a lower-level interface that interacts more directly with the underlying core library and which is similar to those of other language bindings like Python, plus various helpers to interact with its model objects such as by plotting their feature importances or converting them to other formats. @@ -62,7 +65,7 @@ model_abserr <- xgboost(x, y, objective = "reg:absoluteerror", nthreads = 1, nro _Note: the objective must match with the type of the "y" response variable - for example, classification objectives for discrete choices require "factor" types, while regression models for real-valued data require "numeric" types._ -# Model parameters +## Model parameters XGBoost models allow a large degree of control over how they are built. By their nature, gradient-boosted decision tree ensembles are able to capture very complex patterns between features in the data and a response variable, which also means they can suffer from overfitting if not controlled appropirately. @@ -105,7 +108,7 @@ xgboost( ) ``` -# Examining model objects +## Examining model objects XGBoost model objects for the most part consist of a pointer to a C++ object where most of the information is held and which is interfaced through the utility functions and methods in the package, but also contains some R attributes that can be retrieved (and new ones added) through `attributes()`: @@ -131,7 +134,7 @@ xgb.importance(model) xgb.model.dt.tree(model) ``` -# Other features +## Other features XGBoost supports many additional features on top of its traditional gradient-boosting framework, including, among others: @@ -143,7 +146,7 @@ XGBoost supports many additional features on top of its traditional gradient-boo See the [online documentation](https://xgboost.readthedocs.io/en/stable/index.html) - particularly the [tutorials section](https://xgboost.readthedocs.io/en/stable/tutorials/index.html) - for a glimpse over further functionalities that XGBoost offers. -# The low-level interface +## The low-level interface In addition to the `xgboost(x, y, ...)` function, XGBoost also provides a lower-level interface for creating model objects through the function `xgb.train()`, which resembles the same `xgb.train` functions in other language bindings of XGBoost. diff --git a/doc/R-package/index.rst b/doc/R-package/index.rst index 2479e11a77f0..b449ee7fe00c 100644 --- a/doc/R-package/index.rst +++ b/doc/R-package/index.rst @@ -15,6 +15,15 @@ Get Started * Checkout the :doc:`Installation Guide ` contains instructions to install xgboost, and :doc:`Tutorials ` for examples on how to use XGBoost for various tasks. * Read the `API documentation `_. +********* +Vignettes +********* + +.. toctree:: + + xgboost_introduction + xgboostfromJSON + ************ Other topics ************ diff --git a/doc/conf.py b/doc/conf.py index 89dc0f4eaee2..782e5409f266 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,7 +12,6 @@ # All configuration values have a default; values that are commented out # serve to show the default. import os -import re import shutil import subprocess import sys @@ -35,7 +34,7 @@ release = xgboost.__version__ -def run_doxygen(): +def run_doxygen() -> None: """Run the doxygen make command in the designated folder.""" curdir = os.path.normpath(os.path.abspath(os.path.curdir)) if os.path.exists(TMP_DIR): @@ -67,8 +66,8 @@ def run_doxygen(): os.chdir(curdir) -def build_jvm_docs(): - """Build docs for the JVM packages""" +def get_branch() -> str: + """Guess the git branch.""" git_branch = os.getenv("READTHEDOCS_VERSION_NAME", default=None) print(f"READTHEDOCS_VERSION_NAME = {git_branch}") @@ -79,6 +78,12 @@ def build_jvm_docs(): elif git_branch == "stable": git_branch = f"release_{xgboost.__version__}" print(f"git_branch = {git_branch}") + return git_branch + + +def build_jvm_docs() -> None: + """Build docs for the JVM packages""" + git_branch = get_branch() def try_fetch_jvm_doc(branch): """ @@ -106,10 +111,37 @@ def try_fetch_jvm_doc(branch): return False if not try_fetch_jvm_doc(git_branch): - print(f"Falling back to the master branch...") + print("Falling back to the master branch...") try_fetch_jvm_doc("master") +def build_r_docs() -> None: + """Fetch R document from s3.""" + git_branch = get_branch() + + def try_fetch_r_doc(branch: str) -> bool: + try: + url = f"https://s3-us-west-2.amazonaws.com/xgboost-docs/r-docs-{branch}.tar.bz2" + filename, _ = urllib.request.urlretrieve(url) + if not os.path.exists(TMP_DIR): + print(f"Create directory {TMP_DIR}") + os.mkdir(TMP_DIR) + r_doc_dir = os.path.join(TMP_DIR, "r_docs") + if os.path.exists(r_doc_dir): + shutil.rmtree(r_doc_dir) + os.mkdir(r_doc_dir) + + with tarfile.open(filename, "r:bz2") as t: + t.extractall(r_doc_dir) + return True + except HTTPError: + print(f"R doc not found at {url}.") + return False + + if not try_fetch_r_doc(git_branch): + try_fetch_r_doc("master") + + def is_readthedocs_build(): if os.environ.get("READTHEDOCS", None) == "True": return True @@ -125,6 +157,7 @@ def is_readthedocs_build(): if is_readthedocs_build(): run_doxygen() build_jvm_docs() + build_r_docs() # If extensions (or modules to document with autodoc) are in another directory, diff --git a/ops/pipeline/build-r-docs-impl.sh b/ops/pipeline/build-r-docs-impl.sh new file mode 100644 index 000000000000..d9f5eb6bf0bc --- /dev/null +++ b/ops/pipeline/build-r-docs-impl.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if [[ $# -ne 1 ]] +then + echo "Usage: $0 [branch name]" + exit 1 +fi + +set -euo pipefail + +branch_name=$1 + +# See instructions at: https://cran.r-project.org/bin/linux/ubuntu/ + +wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc +# add the R 4.0 repo from CRAN -- adjust 'focal' to 'groovy' or 'bionic' as needed +sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" + +sudo apt install --no-install-recommends r-base +Rscript -e "install.packages(c('pkgdown'), repos = 'https://mirror.las.iastate.edu/CRAN/')" +cd R-package +Rscript -e "pkgdown::build_site()" +cd - +tar cvjf r-docs-${branch_name}.tar.bz2 R-package/docs diff --git a/ops/pipeline/build-r-docs.sh b/ops/pipeline/build-r-docs.sh new file mode 100644 index 000000000000..61ae16e92ecc --- /dev/null +++ b/ops/pipeline/build-r-docs.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -euo pipefail + +if [[ -z ${BRANCH_NAME:-} ]] +then + echo "Make sure to define environment variable BRANCH_NAME." + exit 1 +fi + +source ops/pipeline/get-docker-registry-details.sh + +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu + +echo "--- Build R package doc" +set -x +python3 ops/docker_run.py \ + --image-uri ${IMAGE_URI} \ + -- ops/pipeline/build-r-docs-impl.sh ${BRANCH_NAME} From bf8250b3b64ac6a2cc3c5f59eeb76da1f64be07b Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Thu, 16 Jan 2025 15:48:22 +0800 Subject: [PATCH 02/42] use r image. [skip ci] --- doc/R-package/.gitignore | 1 + doc/R-package/Makefile | 2 ++ ops/pipeline/build-r-docs-impl.sh | 17 +++++++++-------- ops/pipeline/build-r-docs.sh | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) mode change 100644 => 100755 ops/pipeline/build-r-docs-impl.sh mode change 100644 => 100755 ops/pipeline/build-r-docs.sh diff --git a/doc/R-package/.gitignore b/doc/R-package/.gitignore index b25c15b81fae..8a51ab19c717 100644 --- a/doc/R-package/.gitignore +++ b/doc/R-package/.gitignore @@ -1 +1,2 @@ *~ +*.md diff --git a/doc/R-package/Makefile b/doc/R-package/Makefile index c9a3a494404e..5eff72863f43 100644 --- a/doc/R-package/Makefile +++ b/doc/R-package/Makefile @@ -5,6 +5,8 @@ PKGROOT=../../R-package xgboostfromJSON.md: $(PKGROOT)/vignettes/xgboostfromJSON.Rmd xgboost_introduction.md: $(PKGROOT)/vignettes/xgboost_introduction.Rmd +all: xgboostfromJSON.md xgboost_introduction.md + # General Rules for build rmarkdowns, need knitr %.md: Rscript -e \ diff --git a/ops/pipeline/build-r-docs-impl.sh b/ops/pipeline/build-r-docs-impl.sh old mode 100644 new mode 100755 index d9f5eb6bf0bc..bcedd5c00031 --- a/ops/pipeline/build-r-docs-impl.sh +++ b/ops/pipeline/build-r-docs-impl.sh @@ -10,15 +10,16 @@ set -euo pipefail branch_name=$1 -# See instructions at: https://cran.r-project.org/bin/linux/ubuntu/ +echo "R_LIBS_USER: ${R_LIBS_USER}" -wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc -# add the R 4.0 repo from CRAN -- adjust 'focal' to 'groovy' or 'bionic' as needed -sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" +gosu root chown -R $UID:$GROUPS ${R_LIBS_USER} -sudo apt install --no-install-recommends r-base -Rscript -e "install.packages(c('pkgdown'), repos = 'https://mirror.las.iastate.edu/CRAN/')" cd R-package -Rscript -e "pkgdown::build_site()" +MAKEFLAGS=-j$(nproc) Rscript ./tests/helper_scripts/install_deps.R +MAKEFLAGS=-j$(nproc) Rscript -e "pkgdown::build_site()" cd - -tar cvjf r-docs-${branch_name}.tar.bz2 R-package/docs + +cd doc/R-package +make -j$(nproc) all + +tar cvjf r-docs-${branch_name}.tar.bz2 R-package/docs doc/R-package/xgboost_introduction.md doc/R-package/xgboostfromJSON.md diff --git a/ops/pipeline/build-r-docs.sh b/ops/pipeline/build-r-docs.sh old mode 100644 new mode 100755 index 61ae16e92ecc..09147415ecfe --- a/ops/pipeline/build-r-docs.sh +++ b/ops/pipeline/build-r-docs.sh @@ -10,7 +10,7 @@ fi source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8 echo "--- Build R package doc" set -x From 2ed6ba80c21ba52ecf23831a69153451216a0f75 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Thu, 16 Jan 2025 21:37:14 +0800 Subject: [PATCH 03/42] disable for now --- ops/pipeline/build-r-docs-impl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/pipeline/build-r-docs-impl.sh b/ops/pipeline/build-r-docs-impl.sh index bcedd5c00031..989150acd854 100755 --- a/ops/pipeline/build-r-docs-impl.sh +++ b/ops/pipeline/build-r-docs-impl.sh @@ -15,7 +15,7 @@ echo "R_LIBS_USER: ${R_LIBS_USER}" gosu root chown -R $UID:$GROUPS ${R_LIBS_USER} cd R-package -MAKEFLAGS=-j$(nproc) Rscript ./tests/helper_scripts/install_deps.R +# MAKEFLAGS=-j$(nproc) Rscript ./tests/helper_scripts/install_deps.R MAKEFLAGS=-j$(nproc) Rscript -e "pkgdown::build_site()" cd - From 97609e68ff3addaf72c520c5e51504f8a2bbefaa Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Thu, 16 Jan 2025 22:23:27 +0800 Subject: [PATCH 04/42] skip examples [skip ci] --- .github/workflows/r_tests.yml | 2 +- doc/R-package/index.rst | 3 ++- ops/pipeline/build-r-docs-impl.sh | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml index 57131b96b96d..e7f53e01fdd0 100644 --- a/.github/workflows/r_tests.yml +++ b/.github/workflows/r_tests.yml @@ -122,4 +122,4 @@ jobs: python3 ops/pipeline/manage-artifacts.py upload \ --s3-bucket xgboost-docs \ --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \ - R-package/r-docs-${{ env.BRANCH_NAME }}.tar.bz2 + r-docs-${{ env.BRANCH_NAME }}.tar.bz2 diff --git a/doc/R-package/index.rst b/doc/R-package/index.rst index b449ee7fe00c..07c0cb2aa10a 100644 --- a/doc/R-package/index.rst +++ b/doc/R-package/index.rst @@ -13,7 +13,8 @@ You have found the XGBoost R Package! Get Started *********** * Checkout the :doc:`Installation Guide ` contains instructions to install xgboost, and :doc:`Tutorials ` for examples on how to use XGBoost for various tasks. -* Read the `API documentation `_. +* Read the `API documentation <>` +* Read the `CRAN documentation `_. ********* Vignettes diff --git a/ops/pipeline/build-r-docs-impl.sh b/ops/pipeline/build-r-docs-impl.sh index 989150acd854..5d9385b35435 100755 --- a/ops/pipeline/build-r-docs-impl.sh +++ b/ops/pipeline/build-r-docs-impl.sh @@ -15,11 +15,14 @@ echo "R_LIBS_USER: ${R_LIBS_USER}" gosu root chown -R $UID:$GROUPS ${R_LIBS_USER} cd R-package -# MAKEFLAGS=-j$(nproc) Rscript ./tests/helper_scripts/install_deps.R -MAKEFLAGS=-j$(nproc) Rscript -e "pkgdown::build_site()" +MAKEFLAGS=-j$(nproc) Rscript ./tests/helper_scripts/install_deps.R +# Some examples are failing +MAKEFLAGS=-j$(nproc) Rscript -e "pkgdown::build_site(examples=FALSE)" cd - cd doc/R-package make -j$(nproc) all +cd ../../ # back to project root + tar cvjf r-docs-${branch_name}.tar.bz2 R-package/docs doc/R-package/xgboost_introduction.md doc/R-package/xgboostfromJSON.md From 2bb1aeac73f7e77a51e621d5469479b40cdb4cf7 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Thu, 16 Jan 2025 22:53:25 +0800 Subject: [PATCH 05/42] witchcraft. [skip ci] --- doc/R-package/index.rst | 2 +- doc/R-package/r_docs/index.rst | 5 +++++ doc/conf.py | 16 ++++++++++++++-- doc/jvm/javadocs/index.rst | 1 + 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 doc/R-package/r_docs/index.rst diff --git a/doc/R-package/index.rst b/doc/R-package/index.rst index 07c0cb2aa10a..7a8e189ce76e 100644 --- a/doc/R-package/index.rst +++ b/doc/R-package/index.rst @@ -13,7 +13,7 @@ You have found the XGBoost R Package! Get Started *********** * Checkout the :doc:`Installation Guide ` contains instructions to install xgboost, and :doc:`Tutorials ` for examples on how to use XGBoost for various tasks. -* Read the `API documentation <>` +* Read the `API documentation <../r_docs/R-package/docs/index.html>`__ * Read the `CRAN documentation `_. ********* diff --git a/doc/R-package/r_docs/index.rst b/doc/R-package/r_docs/index.rst new file mode 100644 index 000000000000..88a8b72879e2 --- /dev/null +++ b/doc/R-package/r_docs/index.rst @@ -0,0 +1,5 @@ +:orphan: + +============= +XGBoost R API +============= diff --git a/doc/conf.py b/doc/conf.py index 782e5409f266..25d622b11dea 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -23,6 +23,7 @@ CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) PROJECT_ROOT = os.path.normpath(os.path.join(CURR_PATH, os.path.pardir)) TMP_DIR = os.path.join(CURR_PATH, "tmp") +print("current path:", CURR_PATH) DOX_DIR = "doxygen" os.environ["XGBOOST_BUILD_DOC"] = "1" @@ -81,6 +82,17 @@ def get_branch() -> str: return git_branch +# Witchcraft alert: +# +# Both the jvm and the R document is built with an independent CI pipeline and fetched +# during document build. +# +# The fetched artifacts are stored in xgboost/doc/tmp/jvm_docs and +# xgboost/doc/tmp/r_docs respectively. For the R package, there's a dummy index file in +# xgboost/doc/R-package/r_docs . The xgboost/doc/tmp is part of the `html_extra_path` +# sphinx configuration, which somehow makes sphinx to copy the extracted html files to +# the build directory. + def build_jvm_docs() -> None: """Build docs for the JVM packages""" git_branch = get_branch() @@ -111,7 +123,7 @@ def try_fetch_jvm_doc(branch): return False if not try_fetch_jvm_doc(git_branch): - print("Falling back to the master branch...") + print("Falling back to the master branch.") try_fetch_jvm_doc("master") @@ -135,7 +147,7 @@ def try_fetch_r_doc(branch: str) -> bool: t.extractall(r_doc_dir) return True except HTTPError: - print(f"R doc not found at {url}.") + print(f"R doc not found at {url}. Falling back to the master branch.") return False if not try_fetch_r_doc(git_branch): diff --git a/doc/jvm/javadocs/index.rst b/doc/jvm/javadocs/index.rst index 33bf528129e4..3ff7b882136b 100644 --- a/doc/jvm/javadocs/index.rst +++ b/doc/jvm/javadocs/index.rst @@ -1,3 +1,4 @@ +:orphan: ================== XGBoost4J Java API ================== From 1c41cb7a30454426e89dcbcb89b58ce8e726448b Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Thu, 16 Jan 2025 23:12:44 +0800 Subject: [PATCH 06/42] vig. --- doc/conf.py | 8 ++++++++ doc/jvm/javadocs/index.rst | 1 + 2 files changed, 9 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index 25d622b11dea..d6a42304eda3 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -93,6 +93,7 @@ def get_branch() -> str: # sphinx configuration, which somehow makes sphinx to copy the extracted html files to # the build directory. + def build_jvm_docs() -> None: """Build docs for the JVM packages""" git_branch = get_branch() @@ -145,6 +146,13 @@ def try_fetch_r_doc(branch: str) -> bool: with tarfile.open(filename, "r:bz2") as t: t.extractall(r_doc_dir) + + for root, subdir, files in os.walk(os.path.join(r_doc_dir, "doc", "R-package")): + for f in files: + assert f.endswith(".md") + src = os.path.join(root, f) + dst = os.path.join(PROJECT_ROOT, "doc", "R-package", f) + shutil.move(src, dst) return True except HTTPError: print(f"R doc not found at {url}. Falling back to the master branch.") diff --git a/doc/jvm/javadocs/index.rst b/doc/jvm/javadocs/index.rst index 3ff7b882136b..c6e38af94eca 100644 --- a/doc/jvm/javadocs/index.rst +++ b/doc/jvm/javadocs/index.rst @@ -1,4 +1,5 @@ :orphan: + ================== XGBoost4J Java API ================== From 74d3fcb6a941981dc2b6643725464ed47c86cd0a Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 00:14:20 +0800 Subject: [PATCH 07/42] ref. --- .github/workflows/r_tests.yml | 6 +++++- ops/pipeline/build-r-docs.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml index e7f53e01fdd0..e2665846fea9 100644 --- a/.github/workflows/r_tests.yml +++ b/.github/workflows/r_tests.yml @@ -12,6 +12,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + BRANCH_NAME: >- + ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} + jobs: test-Rpkg: runs-on: ${{ matrix.os }} @@ -107,7 +111,7 @@ jobs: runs-on: - runs-on=${{ github.run_id }} - runner=linux-amd64-cpu - - tag=r-tests-build-jvm-docs + - tag=r-tests-build-docs steps: # Restart Docker daemon so that it recognizes the ephemeral disks - run: sudo systemctl restart docker diff --git a/ops/pipeline/build-r-docs.sh b/ops/pipeline/build-r-docs.sh index 09147415ecfe..b6235a9b1be7 100755 --- a/ops/pipeline/build-r-docs.sh +++ b/ops/pipeline/build-r-docs.sh @@ -10,7 +10,7 @@ fi source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu_build_r_docs:PR-6 echo "--- Build R package doc" set -x From f9a359ca9dd88cede70f4f931c297f6a0bf4b812 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 00:16:10 +0800 Subject: [PATCH 08/42] ref. --- ops/pipeline/build-r-docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/pipeline/build-r-docs.sh b/ops/pipeline/build-r-docs.sh index b6235a9b1be7..1a123d5fef81 100755 --- a/ops/pipeline/build-r-docs.sh +++ b/ops/pipeline/build-r-docs.sh @@ -10,7 +10,7 @@ fi source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu_build_r_docs:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu_build_r_doc:PR-6 echo "--- Build R package doc" set -x From f283e3cd7fdadfbd65cb3913e08e1a2b9d621efb Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 00:24:03 +0800 Subject: [PATCH 09/42] ref. --- .github/workflows/r_tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml index e2665846fea9..273a850256eb 100644 --- a/.github/workflows/r_tests.yml +++ b/.github/workflows/r_tests.yml @@ -4,6 +4,8 @@ on: [push, pull_request] env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + BRANCH_NAME: >- + ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} permissions: contents: read # to fetch code (actions/checkout) @@ -12,9 +14,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -env: - BRANCH_NAME: >- - ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} jobs: test-Rpkg: From a280f90d229fb4377f586787d447aae91304cd0d Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 00:36:00 +0800 Subject: [PATCH 10/42] all? --- ops/pipeline/build-cpu-arm64.sh | 2 +- ops/pipeline/build-cpu.sh | 2 +- ops/pipeline/build-cuda.sh | 4 ++-- ops/pipeline/build-gpu-rpkg.sh | 2 +- ops/pipeline/build-jvm-doc.sh | 2 +- ops/pipeline/build-jvm-gpu.sh | 2 +- ops/pipeline/build-jvm-manylinux2014.sh | 2 +- ops/pipeline/build-manylinux2014.sh | 2 +- ops/pipeline/build-test-jvm-packages.sh | 2 +- ops/pipeline/deploy-jvm-packages.sh | 2 +- ops/pipeline/run-clang-tidy.sh | 2 +- ops/pipeline/test-cpp-gpu.sh | 2 +- ops/pipeline/test-cpp-i386.sh | 2 +- ops/pipeline/test-jvm-gpu.sh | 2 +- ops/pipeline/test-python-wheel.sh | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ops/pipeline/build-cpu-arm64.sh b/ops/pipeline/build-cpu-arm64.sh index e3e0cebb22c8..c52777ffdef7 100755 --- a/ops/pipeline/build-cpu-arm64.sh +++ b/ops/pipeline/build-cpu-arm64.sh @@ -13,7 +13,7 @@ source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG=manylinux_2_28_aarch64 -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.aarch64:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.aarch64:PR-6 echo "--- Build CPU code targeting ARM64" set -x diff --git a/ops/pipeline/build-cpu.sh b/ops/pipeline/build-cpu.sh index 2443006bd7c2..637546d9ccdd 100755 --- a/ops/pipeline/build-cpu.sh +++ b/ops/pipeline/build-cpu.sh @@ -6,7 +6,7 @@ set -euo pipefail source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu:PR-6 echo "--- Build CPU code" set -x diff --git a/ops/pipeline/build-cuda.sh b/ops/pipeline/build-cuda.sh index a34a0dcc7c28..0f8220e0d93b 100755 --- a/ops/pipeline/build-cuda.sh +++ b/ops/pipeline/build-cuda.sh @@ -35,8 +35,8 @@ source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG=manylinux_2_28_x86_64 -BUILD_IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" -MANYLINUX_IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.${WHEEL_TAG}:main" +BUILD_IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:PR-6" +MANYLINUX_IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.${WHEEL_TAG}:PR-6" echo "--- Build with CUDA" diff --git a/ops/pipeline/build-gpu-rpkg.sh b/ops/pipeline/build-gpu-rpkg.sh index ced68245d4bf..8860ced3121e 100755 --- a/ops/pipeline/build-gpu-rpkg.sh +++ b/ops/pipeline/build-gpu-rpkg.sh @@ -11,7 +11,7 @@ fi source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:PR-6 echo "--- Build XGBoost R package with CUDA" set -x diff --git a/ops/pipeline/build-jvm-doc.sh b/ops/pipeline/build-jvm-doc.sh index 3f9fad3bc2d0..a383fc37cc10 100755 --- a/ops/pipeline/build-jvm-doc.sh +++ b/ops/pipeline/build-jvm-doc.sh @@ -19,7 +19,7 @@ fi source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:PR-6 echo "--- Build JVM packages doc" set -x diff --git a/ops/pipeline/build-jvm-gpu.sh b/ops/pipeline/build-jvm-gpu.sh index 2f71849efb3b..8812bdece95d 100755 --- a/ops/pipeline/build-jvm-gpu.sh +++ b/ops/pipeline/build-jvm-gpu.sh @@ -6,7 +6,7 @@ set -euo pipefail source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:PR-6 echo "--- Build libxgboost4j.so with CUDA" diff --git a/ops/pipeline/build-jvm-manylinux2014.sh b/ops/pipeline/build-jvm-manylinux2014.sh index 5b3942fb67ef..5cea3973fa48 100755 --- a/ops/pipeline/build-jvm-manylinux2014.sh +++ b/ops/pipeline/build-jvm-manylinux2014.sh @@ -15,7 +15,7 @@ image_repo="xgb-ci.manylinux2014_${arch}" source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:PR-6" # Build XGBoost4J binary echo "--- Build libxgboost4j.so (targeting glibc 2.17)" diff --git a/ops/pipeline/build-manylinux2014.sh b/ops/pipeline/build-manylinux2014.sh index cf8ca7a94da3..73de518e3ea4 100755 --- a/ops/pipeline/build-manylinux2014.sh +++ b/ops/pipeline/build-manylinux2014.sh @@ -21,7 +21,7 @@ source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG="manylinux2014_${arch}" IMAGE_REPO="xgb-ci.${WHEEL_TAG}" -IMAGE_URI="${DOCKER_REGISTRY_URL}/${IMAGE_REPO}:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${IMAGE_REPO}:PR-6" PYTHON_BIN="/opt/python/cp310-cp310/bin/python" echo "--- Build binary wheel for ${WHEEL_TAG}" diff --git a/ops/pipeline/build-test-jvm-packages.sh b/ops/pipeline/build-test-jvm-packages.sh index 5c1f5e8b0d4a..7ffcd00aaaf8 100755 --- a/ops/pipeline/build-test-jvm-packages.sh +++ b/ops/pipeline/build-test-jvm-packages.sh @@ -23,7 +23,7 @@ do fi done -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm:PR-6 set -x diff --git a/ops/pipeline/deploy-jvm-packages.sh b/ops/pipeline/deploy-jvm-packages.sh index d73b289b165d..ee9969c1d4f0 100755 --- a/ops/pipeline/deploy-jvm-packages.sh +++ b/ops/pipeline/deploy-jvm-packages.sh @@ -16,7 +16,7 @@ variant="$1" image_repo="$2" scala_version="$3" -IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:PR-6" set -x diff --git a/ops/pipeline/run-clang-tidy.sh b/ops/pipeline/run-clang-tidy.sh index c6166e99cb33..39693e9e2c65 100755 --- a/ops/pipeline/run-clang-tidy.sh +++ b/ops/pipeline/run-clang-tidy.sh @@ -4,7 +4,7 @@ set -euo pipefail source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.clang_tidy:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.clang_tidy:PR-6 echo "--- Run clang-tidy" set -x diff --git a/ops/pipeline/test-cpp-gpu.sh b/ops/pipeline/test-cpp-gpu.sh index 4262ebda7268..cd809f6a27a6 100755 --- a/ops/pipeline/test-cpp-gpu.sh +++ b/ops/pipeline/test-cpp-gpu.sh @@ -11,7 +11,7 @@ suite=$1 source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu:PR-6 case "${suite}" in gpu) diff --git a/ops/pipeline/test-cpp-i386.sh b/ops/pipeline/test-cpp-i386.sh index dd7827ff2de9..46246c4153db 100755 --- a/ops/pipeline/test-cpp-i386.sh +++ b/ops/pipeline/test-cpp-i386.sh @@ -5,7 +5,7 @@ set -euo pipefail source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.i386:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.i386:PR-6" set -x python3 ops/docker_run.py \ diff --git a/ops/pipeline/test-jvm-gpu.sh b/ops/pipeline/test-jvm-gpu.sh index c7dfdd05b43f..21ebe86c8d54 100755 --- a/ops/pipeline/test-jvm-gpu.sh +++ b/ops/pipeline/test-jvm-gpu.sh @@ -25,7 +25,7 @@ done source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:PR-6 set -x diff --git a/ops/pipeline/test-python-wheel.sh b/ops/pipeline/test-python-wheel.sh index 5551870e0b77..4a403c93179f 100755 --- a/ops/pipeline/test-python-wheel.sh +++ b/ops/pipeline/test-python-wheel.sh @@ -20,7 +20,7 @@ else fi source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:PR-6" set -x python3 ops/docker_run.py --image-uri "${IMAGE_URI}" ${gpu_option} \ From 000472f37e487da0248871998ad1881be504b4fd Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 00:38:27 +0800 Subject: [PATCH 11/42] Revert "all?" This reverts commit 6208a1a88128ddc7187ccd277f2358c600c7a700. --- ops/pipeline/build-cpu-arm64.sh | 2 +- ops/pipeline/build-cpu.sh | 2 +- ops/pipeline/build-cuda.sh | 4 ++-- ops/pipeline/build-gpu-rpkg.sh | 2 +- ops/pipeline/build-jvm-doc.sh | 2 +- ops/pipeline/build-jvm-gpu.sh | 2 +- ops/pipeline/build-jvm-manylinux2014.sh | 2 +- ops/pipeline/build-manylinux2014.sh | 2 +- ops/pipeline/build-test-jvm-packages.sh | 2 +- ops/pipeline/deploy-jvm-packages.sh | 2 +- ops/pipeline/run-clang-tidy.sh | 2 +- ops/pipeline/test-cpp-gpu.sh | 2 +- ops/pipeline/test-cpp-i386.sh | 2 +- ops/pipeline/test-jvm-gpu.sh | 2 +- ops/pipeline/test-python-wheel.sh | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ops/pipeline/build-cpu-arm64.sh b/ops/pipeline/build-cpu-arm64.sh index c52777ffdef7..e3e0cebb22c8 100755 --- a/ops/pipeline/build-cpu-arm64.sh +++ b/ops/pipeline/build-cpu-arm64.sh @@ -13,7 +13,7 @@ source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG=manylinux_2_28_aarch64 -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.aarch64:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.aarch64:main echo "--- Build CPU code targeting ARM64" set -x diff --git a/ops/pipeline/build-cpu.sh b/ops/pipeline/build-cpu.sh index 637546d9ccdd..2443006bd7c2 100755 --- a/ops/pipeline/build-cpu.sh +++ b/ops/pipeline/build-cpu.sh @@ -6,7 +6,7 @@ set -euo pipefail source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu:main echo "--- Build CPU code" set -x diff --git a/ops/pipeline/build-cuda.sh b/ops/pipeline/build-cuda.sh index 0f8220e0d93b..a34a0dcc7c28 100755 --- a/ops/pipeline/build-cuda.sh +++ b/ops/pipeline/build-cuda.sh @@ -35,8 +35,8 @@ source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG=manylinux_2_28_x86_64 -BUILD_IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:PR-6" -MANYLINUX_IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.${WHEEL_TAG}:PR-6" +BUILD_IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" +MANYLINUX_IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.${WHEEL_TAG}:main" echo "--- Build with CUDA" diff --git a/ops/pipeline/build-gpu-rpkg.sh b/ops/pipeline/build-gpu-rpkg.sh index 8860ced3121e..ced68245d4bf 100755 --- a/ops/pipeline/build-gpu-rpkg.sh +++ b/ops/pipeline/build-gpu-rpkg.sh @@ -11,7 +11,7 @@ fi source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:main echo "--- Build XGBoost R package with CUDA" set -x diff --git a/ops/pipeline/build-jvm-doc.sh b/ops/pipeline/build-jvm-doc.sh index a383fc37cc10..3f9fad3bc2d0 100755 --- a/ops/pipeline/build-jvm-doc.sh +++ b/ops/pipeline/build-jvm-doc.sh @@ -19,7 +19,7 @@ fi source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main echo "--- Build JVM packages doc" set -x diff --git a/ops/pipeline/build-jvm-gpu.sh b/ops/pipeline/build-jvm-gpu.sh index 8812bdece95d..2f71849efb3b 100755 --- a/ops/pipeline/build-jvm-gpu.sh +++ b/ops/pipeline/build-jvm-gpu.sh @@ -6,7 +6,7 @@ set -euo pipefail source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main echo "--- Build libxgboost4j.so with CUDA" diff --git a/ops/pipeline/build-jvm-manylinux2014.sh b/ops/pipeline/build-jvm-manylinux2014.sh index 5cea3973fa48..5b3942fb67ef 100755 --- a/ops/pipeline/build-jvm-manylinux2014.sh +++ b/ops/pipeline/build-jvm-manylinux2014.sh @@ -15,7 +15,7 @@ image_repo="xgb-ci.manylinux2014_${arch}" source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:PR-6" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" # Build XGBoost4J binary echo "--- Build libxgboost4j.so (targeting glibc 2.17)" diff --git a/ops/pipeline/build-manylinux2014.sh b/ops/pipeline/build-manylinux2014.sh index 73de518e3ea4..cf8ca7a94da3 100755 --- a/ops/pipeline/build-manylinux2014.sh +++ b/ops/pipeline/build-manylinux2014.sh @@ -21,7 +21,7 @@ source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG="manylinux2014_${arch}" IMAGE_REPO="xgb-ci.${WHEEL_TAG}" -IMAGE_URI="${DOCKER_REGISTRY_URL}/${IMAGE_REPO}:PR-6" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${IMAGE_REPO}:main" PYTHON_BIN="/opt/python/cp310-cp310/bin/python" echo "--- Build binary wheel for ${WHEEL_TAG}" diff --git a/ops/pipeline/build-test-jvm-packages.sh b/ops/pipeline/build-test-jvm-packages.sh index 7ffcd00aaaf8..5c1f5e8b0d4a 100755 --- a/ops/pipeline/build-test-jvm-packages.sh +++ b/ops/pipeline/build-test-jvm-packages.sh @@ -23,7 +23,7 @@ do fi done -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm:main set -x diff --git a/ops/pipeline/deploy-jvm-packages.sh b/ops/pipeline/deploy-jvm-packages.sh index ee9969c1d4f0..d73b289b165d 100755 --- a/ops/pipeline/deploy-jvm-packages.sh +++ b/ops/pipeline/deploy-jvm-packages.sh @@ -16,7 +16,7 @@ variant="$1" image_repo="$2" scala_version="$3" -IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:PR-6" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" set -x diff --git a/ops/pipeline/run-clang-tidy.sh b/ops/pipeline/run-clang-tidy.sh index 39693e9e2c65..c6166e99cb33 100755 --- a/ops/pipeline/run-clang-tidy.sh +++ b/ops/pipeline/run-clang-tidy.sh @@ -4,7 +4,7 @@ set -euo pipefail source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.clang_tidy:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.clang_tidy:main echo "--- Run clang-tidy" set -x diff --git a/ops/pipeline/test-cpp-gpu.sh b/ops/pipeline/test-cpp-gpu.sh index cd809f6a27a6..4262ebda7268 100755 --- a/ops/pipeline/test-cpp-gpu.sh +++ b/ops/pipeline/test-cpp-gpu.sh @@ -11,7 +11,7 @@ suite=$1 source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu:main case "${suite}" in gpu) diff --git a/ops/pipeline/test-cpp-i386.sh b/ops/pipeline/test-cpp-i386.sh index 46246c4153db..dd7827ff2de9 100755 --- a/ops/pipeline/test-cpp-i386.sh +++ b/ops/pipeline/test-cpp-i386.sh @@ -5,7 +5,7 @@ set -euo pipefail source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.i386:PR-6" +IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.i386:main" set -x python3 ops/docker_run.py \ diff --git a/ops/pipeline/test-jvm-gpu.sh b/ops/pipeline/test-jvm-gpu.sh index 21ebe86c8d54..c7dfdd05b43f 100755 --- a/ops/pipeline/test-jvm-gpu.sh +++ b/ops/pipeline/test-jvm-gpu.sh @@ -25,7 +25,7 @@ done source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main set -x diff --git a/ops/pipeline/test-python-wheel.sh b/ops/pipeline/test-python-wheel.sh index 4a403c93179f..5551870e0b77 100755 --- a/ops/pipeline/test-python-wheel.sh +++ b/ops/pipeline/test-python-wheel.sh @@ -20,7 +20,7 @@ else fi source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:PR-6" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" set -x python3 ops/docker_run.py --image-uri "${IMAGE_URI}" ${gpu_option} \ From 239f7c56b42714f15aaaf6f0e2b97b0ec632fcc9 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 00:58:01 +0800 Subject: [PATCH 12/42] note. --- doc/conf.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index d6a42304eda3..4eba90108619 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -23,7 +23,6 @@ CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) PROJECT_ROOT = os.path.normpath(os.path.join(CURR_PATH, os.path.pardir)) TMP_DIR = os.path.join(CURR_PATH, "tmp") -print("current path:", CURR_PATH) DOX_DIR = "doxygen" os.environ["XGBOOST_BUILD_DOC"] = "1" @@ -89,13 +88,15 @@ def get_branch() -> str: # # The fetched artifacts are stored in xgboost/doc/tmp/jvm_docs and # xgboost/doc/tmp/r_docs respectively. For the R package, there's a dummy index file in -# xgboost/doc/R-package/r_docs . The xgboost/doc/tmp is part of the `html_extra_path` -# sphinx configuration, which somehow makes sphinx to copy the extracted html files to -# the build directory. +# xgboost/doc/R-package/r_docs . Jvm doc is similar. As for the C doc, it's generated +# using doxygen and processed by breathe. The generated xml files are stored in +# xgboost/doc/tmp/dev . The xgboost/doc/tmp is part of the `html_extra_path` sphinx +# configuration, which somehow makes sphinx to copy the extracted html files to the +# build directory. def build_jvm_docs() -> None: - """Build docs for the JVM packages""" + """Fetch docs for the JVM packages""" git_branch = get_branch() def try_fetch_jvm_doc(branch): From 2a59244a46249f64c2961dcb36305ee37e61a719 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 01:58:13 +0800 Subject: [PATCH 13/42] Different xgboost. --- doc/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index ddff9be928b6..1fb5fef050dd 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -10,7 +10,6 @@ numpy scipy myst-parser ray[train] -xgboost_ray sphinx-gallery pyspark cloudpickle From 3b22075eb0e3a00d021ef9f71b9c0acb2650cb32 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 02:11:05 +0800 Subject: [PATCH 14/42] use commit hash. --- .github/workflows/jvm_tests.yml | 3 ++- .github/workflows/r_tests.yml | 3 ++- doc/conf.py | 22 ++++++++++++++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml index fc2f21d68043..d88381146fe0 100644 --- a/.github/workflows/jvm_tests.yml +++ b/.github/workflows/jvm_tests.yml @@ -10,6 +10,7 @@ concurrency: cancel-in-progress: true env: + PR_COMMIT_SHA: ${{ github.event.pull_request.head.sha }} BRANCH_NAME: >- ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} @@ -122,7 +123,7 @@ jobs: run: | python3 ops/pipeline/manage-artifacts.py upload \ --s3-bucket xgboost-docs \ - --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \ + --prefix ${BRANCH_NAME}/${PR_COMMIT_SHA} --make-public \ jvm-packages/${{ env.BRANCH_NAME }}.tar.bz2 build-test-jvm-packages: diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml index 273a850256eb..208566e22d17 100644 --- a/.github/workflows/r_tests.yml +++ b/.github/workflows/r_tests.yml @@ -4,6 +4,7 @@ on: [push, pull_request] env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + PR_COMMIT_SHA: ${{ github.event.pull_request.head.sha }} BRANCH_NAME: >- ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} @@ -124,5 +125,5 @@ jobs: run: | python3 ops/pipeline/manage-artifacts.py upload \ --s3-bucket xgboost-docs \ - --prefix ${BRANCH_NAME}/${GITHUB_SHA} --make-public \ + --prefix ${BRANCH_NAME}/${PR_COMMIT_SHA} --make-public \ r-docs-${{ env.BRANCH_NAME }}.tar.bz2 diff --git a/doc/conf.py b/doc/conf.py index 4eba90108619..9c55c1040ed4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -95,9 +95,23 @@ def get_branch() -> str: # build directory. +def get_sha() -> str | None: + sha = os.getenv("READTHEDOCS_GIT_COMMIT_HASH", default=None) + if sha is None: + res = subprocess.run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE) + if res.returncode != 0: + return None + sha = res.stdout.decode("utf-8") + return sha + + def build_jvm_docs() -> None: """Fetch docs for the JVM packages""" git_branch = get_branch() + commit = get_sha() + if commit is None: + print("Couldn't find commit to build jvm docs.") + return def try_fetch_jvm_doc(branch): """ @@ -105,7 +119,7 @@ def try_fetch_jvm_doc(branch): Returns True if successful """ try: - url = f"https://s3-us-west-2.amazonaws.com/xgboost-docs/{branch}.tar.bz2" + url = f"https://xgboost-docs.s3.us-west-2.amazonaws.com/PR-{branch}/{commit}/PR-{branch}.tar.bz2" filename, _ = urllib.request.urlretrieve(url) if not os.path.exists(TMP_DIR): print(f"Create directory {TMP_DIR}") @@ -132,10 +146,14 @@ def try_fetch_jvm_doc(branch): def build_r_docs() -> None: """Fetch R document from s3.""" git_branch = get_branch() + commit = get_sha() + if commit is None: + print("Couldn't find commit to build R docs.") + return def try_fetch_r_doc(branch: str) -> bool: try: - url = f"https://s3-us-west-2.amazonaws.com/xgboost-docs/r-docs-{branch}.tar.bz2" + url = f"https://xgboost-docs.s3.us-west-2.amazonaws.com/PR-{branch}/{commit}/r-docs-PR-{branch}.tar.bz2" filename, _ = urllib.request.urlretrieve(url) if not os.path.exists(TMP_DIR): print(f"Create directory {TMP_DIR}") From aa6fdd2a3aeb7cc20c3c5d8858432029b32ff281 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 02:15:27 +0800 Subject: [PATCH 15/42] add path. --- doc/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index 9c55c1040ed4..b3fdd0f8c1c7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -25,6 +25,8 @@ TMP_DIR = os.path.join(CURR_PATH, "tmp") DOX_DIR = "doxygen" +sys.path.append(os.path.join(PROJECT_ROOT, "python-package")) + os.environ["XGBOOST_BUILD_DOC"] = "1" # Version information. From 5a5cf56523c6ae833c1320e248b2efd82c97b373 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 02:24:02 +0800 Subject: [PATCH 16/42] Note. --- doc/conf.py | 87 ++++++++++++++++++++++++++++---------------- doc/requirements.txt | 1 + 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index b3fdd0f8c1c7..040bcaeb3d24 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -25,8 +25,9 @@ TMP_DIR = os.path.join(CURR_PATH, "tmp") DOX_DIR = "doxygen" +# Directly load the source module. sys.path.append(os.path.join(PROJECT_ROOT, "python-package")) - +# Tell xgboost to not load the libxgboost.so os.environ["XGBOOST_BUILD_DOC"] = "1" # Version information. @@ -36,6 +37,34 @@ release = xgboost.__version__ +# Witchcraft alert: +# +# Both the jvm and the R document is built with an independent CI pipeline and fetched +# during document build. +# +# The fetched artifacts are stored in xgboost/doc/tmp/jvm_docs and +# xgboost/doc/tmp/r_docs respectively. For the R package, there's a dummy index file in +# xgboost/doc/R-package/r_docs . Jvm doc is similar. As for the C doc, it's generated +# using doxygen and processed by breathe during build and there's no independent CI +# pipeline as it's relatively cheap. The generated xml files are stored in +# xgboost/doc/tmp/dev . +# +# The xgboost/doc/tmp is part of the `html_extra_path` sphinx configuration, which +# somehow makes sphinx to copy the extracted html files to the build directory. +# +# There's very limited dependency management in the readthedocs builder, to avoid +# building everything from source using the RTD worker (like packages from CRAN or +# XGBoost itself), we offload to CI workers as much as possible. +# +# However, since there's no way to synchronize the RTD worker with other GitHub workers, +# to see the latest document for a PR branch, we need to re-run RTD manually after +# related doc are uploaded to the S3 bucket. + + +# Document is uploaded to here by the CI builder. +S3_BUCKET = "https://xgboost-docs.s3.us-west-2.amazonaws.com" + + def run_doxygen() -> None: """Run the doxygen make command in the designated folder.""" curdir = os.path.normpath(os.path.abspath(os.path.curdir)) @@ -73,44 +102,36 @@ def get_branch() -> str: git_branch = os.getenv("READTHEDOCS_VERSION_NAME", default=None) print(f"READTHEDOCS_VERSION_NAME = {git_branch}") - if not git_branch: - git_branch = "master" + if not git_branch: # Not in RTD + git_branch = "master" # use the master branch as the default. elif git_branch == "latest": git_branch = "master" elif git_branch == "stable": git_branch = f"release_{xgboost.__version__}" + else: # Likely PR branch + git_branch = f"PR-{git_branch}" print(f"git_branch = {git_branch}") return git_branch -# Witchcraft alert: -# -# Both the jvm and the R document is built with an independent CI pipeline and fetched -# during document build. -# -# The fetched artifacts are stored in xgboost/doc/tmp/jvm_docs and -# xgboost/doc/tmp/r_docs respectively. For the R package, there's a dummy index file in -# xgboost/doc/R-package/r_docs . Jvm doc is similar. As for the C doc, it's generated -# using doxygen and processed by breathe. The generated xml files are stored in -# xgboost/doc/tmp/dev . The xgboost/doc/tmp is part of the `html_extra_path` sphinx -# configuration, which somehow makes sphinx to copy the extracted html files to the -# build directory. - - -def get_sha() -> str | None: +def get_sha(branch: str) -> str | None: sha = os.getenv("READTHEDOCS_GIT_COMMIT_HASH", default=None) - if sha is None: + if sha is not None: + return sha + + if branch == "master": + res = subprocess.run(["git", "rev-parse", "master"], stdout=subprocess.PIPE) + else: res = subprocess.run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE) - if res.returncode != 0: - return None - sha = res.stdout.decode("utf-8") - return sha + if res.returncode != 0: + return None + return res.stdout.decode("utf-8").strip() def build_jvm_docs() -> None: """Fetch docs for the JVM packages""" - git_branch = get_branch() - commit = get_sha() + branch = get_branch() + commit = get_sha(branch) if commit is None: print("Couldn't find commit to build jvm docs.") return @@ -121,7 +142,7 @@ def try_fetch_jvm_doc(branch): Returns True if successful """ try: - url = f"https://xgboost-docs.s3.us-west-2.amazonaws.com/PR-{branch}/{commit}/PR-{branch}.tar.bz2" + url = f"{S3_BUCKET}/{branch}/{commit}/{branch}.tar.bz2" filename, _ = urllib.request.urlretrieve(url) if not os.path.exists(TMP_DIR): print(f"Create directory {TMP_DIR}") @@ -140,22 +161,22 @@ def try_fetch_jvm_doc(branch): print(f"JVM doc not found at {url}. Skipping...") return False - if not try_fetch_jvm_doc(git_branch): + if not try_fetch_jvm_doc(branch): print("Falling back to the master branch.") try_fetch_jvm_doc("master") def build_r_docs() -> None: """Fetch R document from s3.""" - git_branch = get_branch() - commit = get_sha() + branch = get_branch() + commit = get_sha(branch) if commit is None: print("Couldn't find commit to build R docs.") return def try_fetch_r_doc(branch: str) -> bool: try: - url = f"https://xgboost-docs.s3.us-west-2.amazonaws.com/PR-{branch}/{commit}/r-docs-PR-{branch}.tar.bz2" + url = f"{S3_BUCKET}/{branch}/{commit}/r-docs-{branch}.tar.bz2" filename, _ = urllib.request.urlretrieve(url) if not os.path.exists(TMP_DIR): print(f"Create directory {TMP_DIR}") @@ -168,7 +189,9 @@ def try_fetch_r_doc(branch: str) -> bool: with tarfile.open(filename, "r:bz2") as t: t.extractall(r_doc_dir) - for root, subdir, files in os.walk(os.path.join(r_doc_dir, "doc", "R-package")): + for root, subdir, files in os.walk( + os.path.join(r_doc_dir, "doc", "R-package") + ): for f in files: assert f.endswith(".md") src = os.path.join(root, f) @@ -179,7 +202,7 @@ def try_fetch_r_doc(branch: str) -> bool: print(f"R doc not found at {url}. Falling back to the master branch.") return False - if not try_fetch_r_doc(git_branch): + if not try_fetch_r_doc(branch): try_fetch_r_doc("master") diff --git a/doc/requirements.txt b/doc/requirements.txt index 1fb5fef050dd..9a2097035228 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -11,6 +11,7 @@ scipy myst-parser ray[train] sphinx-gallery +dask pyspark cloudpickle setuptools From 6f91180c1606a907e572e6b4e4bbd51118386a75 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 02:50:49 +0800 Subject: [PATCH 17/42] note. --- doc/R-package/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/R-package/index.rst b/doc/R-package/index.rst index 7a8e189ce76e..583caf9fbe65 100644 --- a/doc/R-package/index.rst +++ b/doc/R-package/index.rst @@ -13,7 +13,7 @@ You have found the XGBoost R Package! Get Started *********** * Checkout the :doc:`Installation Guide ` contains instructions to install xgboost, and :doc:`Tutorials ` for examples on how to use XGBoost for various tasks. -* Read the `API documentation <../r_docs/R-package/docs/index.html>`__ +* Read the latest `API documentation <../r_docs/R-package/docs/index.html>`__ . This might refer to a newer version than the one on CRAN. * Read the `CRAN documentation `_. ********* From 6635a1aaa641540d1c723bbef2ef43971f9c026f Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 15:41:49 +0800 Subject: [PATCH 18/42] Install. --- R-package/README.md | 6 ++--- R-package/pkgdown/_pkgdown.yml | 1 + R-package/tests/helper_scripts/install_deps.R | 1 - ops/pipeline/build-r-docs-impl.sh | 25 ++++++++++++++++--- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/R-package/README.md b/R-package/README.md index 50f773755e6f..68a3ffbbf021 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -1,5 +1,5 @@ -XGBoost R Package for Scalable GBM -================================== +XGBoost R Package +================= [![CRAN Status Badge](http://www.r-pkg.org/badges/version/xgboost)](https://cran.r-project.org/web/packages/xgboost) [![CRAN Downloads](http://cranlogs.r-pkg.org/badges/xgboost)](https://cran.rstudio.com/web/packages/xgboost/index.html) @@ -7,7 +7,7 @@ XGBoost R Package for Scalable GBM Resources --------- -* [XGBoost R Package Online Documentation](https://xgboost.readthedocs.org/en/latest/R-package/index.html) +* [XGBoost R Package Online Documentation](https://xgboost.readthedocs.org/en/stable/R-package/index.html) - Check this out for detailed documents, examples and tutorials. Installation diff --git a/R-package/pkgdown/_pkgdown.yml b/R-package/pkgdown/_pkgdown.yml index 3a3c60f97169..d770b44863ff 100644 --- a/R-package/pkgdown/_pkgdown.yml +++ b/R-package/pkgdown/_pkgdown.yml @@ -2,3 +2,4 @@ url: https://github.com/dmlc/xgboost template: bootstrap: 5 + math-rendering: mathjax diff --git a/R-package/tests/helper_scripts/install_deps.R b/R-package/tests/helper_scripts/install_deps.R index 757475541fd4..8bb56b76a576 100644 --- a/R-package/tests/helper_scripts/install_deps.R +++ b/R-package/tests/helper_scripts/install_deps.R @@ -26,7 +26,6 @@ pkgs <- c( "RhpcBLASctl", ## imports "Matrix", - "methods", "data.table", "jsonlite" ) diff --git a/ops/pipeline/build-r-docs-impl.sh b/ops/pipeline/build-r-docs-impl.sh index 5d9385b35435..d61dbb9a2b27 100755 --- a/ops/pipeline/build-r-docs-impl.sh +++ b/ops/pipeline/build-r-docs-impl.sh @@ -6,18 +6,37 @@ then exit 1 fi -set -euo pipefail - branch_name=$1 +if [[ -z "${R_LIBS_USER}" ]]; +then + export R_LIBS_USER=/tmp/rtmpdir +fi + +set -euo pipefail + echo "R_LIBS_USER: ${R_LIBS_USER}" -gosu root chown -R $UID:$GROUPS ${R_LIBS_USER} +if [[ ! -d ${R_LIBS_USER} ]] +then + echo "Make ${R_LIBS_USER} for installing temporary R packages." + mkdir ${R_LIBS_USER} +fi + +# Used only in container environment +if command -v gosu 2>&1 >/dev/null +then + gosu root chown -R $UID:$GROUPS ${R_LIBS_USER} +fi cd R-package + MAKEFLAGS=-j$(nproc) Rscript ./tests/helper_scripts/install_deps.R # Some examples are failing MAKEFLAGS=-j$(nproc) Rscript -e "pkgdown::build_site(examples=FALSE)" +# Install the package for vignettes +MAKEFLAGS=-j$(nproc) R CMD INSTALL . + cd - cd doc/R-package From d59cffc00a53b4b390646af9d0f344572ea7edca Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 18:26:51 +0800 Subject: [PATCH 19/42] Fixes. --- R-package/pkgdown/_pkgdown.yml | 88 +++++++++++++++++++++ doc/R-package/index.rst | 9 ++- doc/conf.py | 28 +++++-- doc/jvm/scaladocs/xgboost4j-flink/index.rst | 2 + doc/jvm/scaladocs/xgboost4j-spark/index.rst | 2 + doc/jvm/scaladocs/xgboost4j/index.rst | 2 + doc/tutorials/ray.rst | 15 ++-- 7 files changed, 128 insertions(+), 18 deletions(-) diff --git a/R-package/pkgdown/_pkgdown.yml b/R-package/pkgdown/_pkgdown.yml index d770b44863ff..2f3cf4871ae8 100644 --- a/R-package/pkgdown/_pkgdown.yml +++ b/R-package/pkgdown/_pkgdown.yml @@ -3,3 +3,91 @@ url: https://github.com/dmlc/xgboost template: bootstrap: 5 math-rendering: mathjax + +reference: + - title: High Level Interface + desc: High level XGBoost interface + contents: + - "xgboost" + - "predict.xgboost" + - "print.xgboost" + - title: Datasets + desc: Test datasets bundled with the R package. + contents: + - "agaricus.train" + - "agaricus.test" + - title: Global Configuration + desc: Global configuration for the XGBoost library. + contents: + - "xgb.config" + - "xgb.set.config" + - "xgb.get.config" + - title: DMatrix + desc: Low-level data storage. + contents: + - "xgb.DMatrix" + - "xgb.DMatrix.hasinfo" + - "xgb.DMatrix.save" + - "dim.xgb.DMatrix" + - "dimnames.xgb.DMatrix" + - "print.xgb.DMatrix" + - "xgb.DataBatch" + - "xgb.DataIter" + - "xgb.get.DMatrix.data" + - "xgb.get.DMatrix.num.non.missing" + - "xgb.ExtMemDMatrix" + - "xgb.QuantileDMatrix.from_iterator" + - "xgb.get.DMatrix.qcut" + - "xgb.slice.DMatrix" + - title: Booster + desc: The model for XGBoost. + contents: + - "a-compatibility-note-for-saveRDS-save" + - "coef.xgb.Booster" + - "getinfo.xgb.Booster" + - "predict.xgb.Booster" + - "print.xgb.Booster" + - "xgb.load" + - "xgb.load.raw" + - "xgb.save" + - "xgb.save.raw" + - "xgb.copy.Booster" + - "xgb.slice.Booster" + - "xgb.get.num.boosted.rounds" + - "xgb.is.same.Booster" + - "xgb.importance" + - "xgb.attr" + - "xgb.create.features" + - "xgb.model.dt.tree" + - "xgb.model.parameters<-" + - "xgb.ggplot.deepness" + - "xgb.dump" + - "variable.names.xgb.Booster" + - "xgb.ggplot.importance" + - "xgb.plot.multi.trees" + - "xgb.plot.shap" + - "xgb.ggplot.shap.summary" + - "xgb.plot.tree" + - "xgb.gblinear.history" + - title: Training Callbacks + desc: Callback functions used for training. + contents: + - "xgb.Callback" + - "xgb.cb.cv.predict" + - "xgb.cb.early.stop" + - "xgb.cb.evaluation.log" + - "xgb.cb.gblinear.history" + - "xgb.cb.print.evaluation" + - "xgb.cb.reset.parameters" + - "xgb.cb.save.model" + - title: Low-level Training Functions + desc: Low-level Training Functions with DMatrix and Booster + contents: + - "xgb.params" + - "xgb.train" + - "xgb.cv" + - "print.xgb.cv.synchronous" + - title: Deprecation Settings + contents: + - "xgboost-deprecated" + - "xgboost-options" diff --git a/doc/R-package/index.rst b/doc/R-package/index.rst index 583caf9fbe65..6f7592862b3f 100644 --- a/doc/R-package/index.rst +++ b/doc/R-package/index.rst @@ -9,11 +9,16 @@ XGBoost R Package You have found the XGBoost R Package! +.. toctree:: + :maxdepth: 2 + :titlesonly: + *********** Get Started *********** + * Checkout the :doc:`Installation Guide ` contains instructions to install xgboost, and :doc:`Tutorials ` for examples on how to use XGBoost for various tasks. -* Read the latest `API documentation <../r_docs/R-package/docs/index.html>`__ . This might refer to a newer version than the one on CRAN. +* Read the latest `API documentation <../r_docs/R-package/docs/reference/index.html>`__ . This might refer to a newer version than the one on CRAN. * Read the `CRAN documentation `_. ********* @@ -30,8 +35,6 @@ Other topics ************ .. toctree:: - :maxdepth: 2 - :titlesonly: Handling of indexable elements Developer guide: parameters from core library diff --git a/doc/conf.py b/doc/conf.py index 040bcaeb3d24..b95a1fa8ee57 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,6 +11,12 @@ # # All configuration values have a default; values that are commented out # serve to show the default. +# +# Envs: +# - READTHEDOCS: Read the docs flag, fetch the R and JVM documents. +# - XGBOOST_R_DOCS: Local path for pre-build R document, used for development. +# - XGBOOST_JVM_DOCS: Local path for pre-build JVM document, used for development. + import os import shutil import subprocess @@ -136,20 +142,23 @@ def build_jvm_docs() -> None: print("Couldn't find commit to build jvm docs.") return - def try_fetch_jvm_doc(branch): + def try_fetch_jvm_doc(branch: str) -> bool: """ Attempt to fetch JVM docs for a given branch. Returns True if successful """ try: - url = f"{S3_BUCKET}/{branch}/{commit}/{branch}.tar.bz2" - filename, _ = urllib.request.urlretrieve(url) + local_jvm_docs = os.environ.get("XGBOOST_JVM_DOCS", None) + if local_jvm_docs is not None: + filename = os.path.expanduser(local_jvm_docs) + else: + url = f"{S3_BUCKET}/{branch}/{commit}/{branch}.tar.bz2" + filename, _ = urllib.request.urlretrieve(url) if not os.path.exists(TMP_DIR): print(f"Create directory {TMP_DIR}") os.mkdir(TMP_DIR) jvm_doc_dir = os.path.join(TMP_DIR, "jvm_docs") if os.path.exists(jvm_doc_dir): - print(f"Delete directory {jvm_doc_dir}") shutil.rmtree(jvm_doc_dir) print(f"Create directory {jvm_doc_dir}") os.mkdir(jvm_doc_dir) @@ -176,16 +185,23 @@ def build_r_docs() -> None: def try_fetch_r_doc(branch: str) -> bool: try: - url = f"{S3_BUCKET}/{branch}/{commit}/r-docs-{branch}.tar.bz2" - filename, _ = urllib.request.urlretrieve(url) + local_r_docs = os.environ.get("XGBOOST_R_DOCS", None) + if local_r_docs is not None: + filename = os.path.expanduser(local_r_docs) + else: + url = f"{S3_BUCKET}/{branch}/{commit}/r-docs-{branch}.tar.bz2" + filename, _ = urllib.request.urlretrieve(url) + if not os.path.exists(TMP_DIR): print(f"Create directory {TMP_DIR}") os.mkdir(TMP_DIR) r_doc_dir = os.path.join(TMP_DIR, "r_docs") if os.path.exists(r_doc_dir): shutil.rmtree(r_doc_dir) + print(f"Create directory {r_doc_dir}") os.mkdir(r_doc_dir) + filename = os.path.join(PROJECT_ROOT, "r-docs-foo.tar.bz2") with tarfile.open(filename, "r:bz2") as t: t.extractall(r_doc_dir) diff --git a/doc/jvm/scaladocs/xgboost4j-flink/index.rst b/doc/jvm/scaladocs/xgboost4j-flink/index.rst index fe35703c0a54..e236c947b5c1 100644 --- a/doc/jvm/scaladocs/xgboost4j-flink/index.rst +++ b/doc/jvm/scaladocs/xgboost4j-flink/index.rst @@ -1,3 +1,5 @@ +:orphan: + ========================= XGBoost4J-Flink Scala API ========================= diff --git a/doc/jvm/scaladocs/xgboost4j-spark/index.rst b/doc/jvm/scaladocs/xgboost4j-spark/index.rst index c702f6e42196..1215f074c572 100644 --- a/doc/jvm/scaladocs/xgboost4j-spark/index.rst +++ b/doc/jvm/scaladocs/xgboost4j-spark/index.rst @@ -1,3 +1,5 @@ +:orphan: + ========================= XGBoost4J-Spark Scala API ========================= diff --git a/doc/jvm/scaladocs/xgboost4j/index.rst b/doc/jvm/scaladocs/xgboost4j/index.rst index 542dbc4d338b..2c19063e9194 100644 --- a/doc/jvm/scaladocs/xgboost4j/index.rst +++ b/doc/jvm/scaladocs/xgboost4j/index.rst @@ -1,3 +1,5 @@ +:orphan: + =================== XGBoost4J Scala API =================== diff --git a/doc/tutorials/ray.rst b/doc/tutorials/ray.rst index f3032c9705a6..1e9bc3fafa43 100644 --- a/doc/tutorials/ray.rst +++ b/doc/tutorials/ray.rst @@ -50,7 +50,7 @@ Using XGBoost-Ray for training and prediction XGBoost-Ray uses the same API as core XGBoost. There are only two differences: 1. Instead of using a ``xgboost.DMatrix``, you'll use a ``xgboost_ray.RayDMatrix`` object -2. There is an additional :class:`ray_params ` parameter that you can use to configure distributed training. +2. There is an additional ``xgboost_ray.RayParams`` parameter that you can use to configure distributed training. Simple training example ----------------------- @@ -88,7 +88,7 @@ and train a binary classifier using two actors. The only differences compared to the non-distributed API are the import statement (``xgboost_ray`` instead of ``xgboost``), using the -``RayDMatrix`` instead of the ``DMatrix``, and passing a :class:`RayParams ` object. +``RayDMatrix`` instead of the ``DMatrix``, and passing a ``xgboost_ray.RayParams`` object. The return object is a regular ``xgboost.Booster`` instance. @@ -115,10 +115,8 @@ will integrate this data in the correct order. The RayParams object ======================== -The ``RayParams`` object is used to configure various settings relating to -the distributed training. - -.. autoclass:: xgboost_ray.RayParams +The ``RayParams`` object is used to configure various settings relating to the distributed +training. Multi GPU training ================== @@ -175,7 +173,7 @@ away for the remainder of the training run, and don't have to wait until it is b In practice this usually leads to a very minor decrease in accuracy but a much shorter training time compared to non-elastic training. -Both training modes can be configured using the respective :class:`RayParams ` +Both training modes can be configured using the respective ``xgboost_ray.RayParams`` parameters. Hyperparameter optimization @@ -190,8 +188,7 @@ a few things you need to do: 1. Put your XGBoost-Ray training call into a function accepting parameter configurations (``train_model`` in the example below). -2. Create a :class:`RayParams ` object (``ray_params`` - in the example below). +2. Create a ``xgboost_ray.RayParams`` object (``ray_params`` in the example below). 3. Define the parameter search space (``config`` dict in the example below). 4. Call ``tune.run()``: * The ``metric`` parameter should contain the metric you'd like to optimize. From 2dd5e6106bd40d7ebd3e52a3636598db5d871fe3 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 19:14:06 +0800 Subject: [PATCH 20/42] release branch. --- doc/conf.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index b95a1fa8ee57..3a4aca1a0dd4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -105,19 +105,22 @@ def run_doxygen() -> None: def get_branch() -> str: """Guess the git branch.""" - git_branch = os.getenv("READTHEDOCS_VERSION_NAME", default=None) - print(f"READTHEDOCS_VERSION_NAME = {git_branch}") - - if not git_branch: # Not in RTD - git_branch = "master" # use the master branch as the default. - elif git_branch == "latest": - git_branch = "master" - elif git_branch == "stable": - git_branch = f"release_{xgboost.__version__}" + branch = os.getenv("READTHEDOCS_VERSION_NAME", default=None) + print(f"READTHEDOCS_VERSION_NAME = {branch}") + + if not branch: # Not in RTD + branch = "master" # use the master branch as the default. + elif branch == "latest": + branch = "master" + elif branch.startswith("release_"): + pass # release branch, like: release_2.1.0 + elif branch == "stable": + branch = f"release_{xgboost.__version__}" else: # Likely PR branch - git_branch = f"PR-{git_branch}" - print(f"git_branch = {git_branch}") - return git_branch + assert str(int(branch)) == branch + branch = f"PR-{branch}" + print(f"branch = {branch}") + return branch def get_sha(branch: str) -> str | None: From 4389d8642834795dc15362bc11d9bb55bfb5a624 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 17 Jan 2025 19:17:32 +0800 Subject: [PATCH 21/42] cleanup. --- doc/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 3a4aca1a0dd4..92fa79f6f43a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -204,7 +204,6 @@ def try_fetch_r_doc(branch: str) -> bool: print(f"Create directory {r_doc_dir}") os.mkdir(r_doc_dir) - filename = os.path.join(PROJECT_ROOT, "r-docs-foo.tar.bz2") with tarfile.open(filename, "r:bz2") as t: t.extractall(r_doc_dir) From e6b59b7c0db047bb94c8487f07e50c7062488e01 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Sat, 18 Jan 2025 01:04:19 +0800 Subject: [PATCH 22/42] registry. --- doc/conf.py | 2 ++ ops/pipeline/build-r-docs.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 92fa79f6f43a..cf47022ac6fa 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -157,6 +157,7 @@ def try_fetch_jvm_doc(branch: str) -> bool: else: url = f"{S3_BUCKET}/{branch}/{commit}/{branch}.tar.bz2" filename, _ = urllib.request.urlretrieve(url) + print(f"Finished: {url} -> {filename}") if not os.path.exists(TMP_DIR): print(f"Create directory {TMP_DIR}") os.mkdir(TMP_DIR) @@ -194,6 +195,7 @@ def try_fetch_r_doc(branch: str) -> bool: else: url = f"{S3_BUCKET}/{branch}/{commit}/r-docs-{branch}.tar.bz2" filename, _ = urllib.request.urlretrieve(url) + print(f"Finished: {url} -> {filename}") if not os.path.exists(TMP_DIR): print(f"Create directory {TMP_DIR}") diff --git a/ops/pipeline/build-r-docs.sh b/ops/pipeline/build-r-docs.sh index 1a123d5fef81..32af3a60fb43 100755 --- a/ops/pipeline/build-r-docs.sh +++ b/ops/pipeline/build-r-docs.sh @@ -10,7 +10,7 @@ fi source ops/pipeline/get-docker-registry-details.sh -IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu_build_r_doc:PR-6 +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu_build_r_doc:main echo "--- Build R package doc" set -x From 77c1fd936142e55f730bf80ce529034b635509f7 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 00:43:18 +0800 Subject: [PATCH 23/42] Try to trigger. --- .github/workflows/doc.yml | 35 +++++++++++++++++++++++++++++++++++ .github/workflows/r_tests.yml | 22 ---------------------- ops/pipeline/trigger-rtd.py | 17 +++++++++++++++++ 3 files changed, 52 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/doc.yml create mode 100644 ops/pipeline/trigger-rtd.py diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 000000000000..0993491ce923 --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,35 @@ +name: XGBoost-docs + +on: [push, pull_request] + + +jobs: + build-r-docs: + name: Build docs for the R package + runs-on: + - runs-on=${{ github.run_id }} + - runner=linux-amd64-cpu + - tag=r-tests-build-docs + steps: + # Restart Docker daemon so that it recognizes the ephemeral disks + - run: sudo systemctl restart docker + - uses: actions/checkout@v4 + with: + submodules: "true" + - name: Log into Docker registry (AWS ECR) + run: bash ops/pipeline/login-docker-registry.sh + - run: bash ops/pipeline/build-r-docs.sh + - name: Upload R doc + run: | + python3 ops/pipeline/manage-artifacts.py upload \ + --s3-bucket xgboost-docs \ + --prefix ${BRANCH_NAME}/${PR_COMMIT_SHA} --make-public \ + r-docs-${{ env.BRANCH_NAME }}.tar.bz2 + + trigger-rtd-build: + needs: build-r-docs + name: Trigger Read The Docs build. + runs-on: ubuntu-latest + steps: + - run: | + python3 ops/pipeline/trigger-rtd.py diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml index 208566e22d17..7a7235cb2df9 100644 --- a/.github/workflows/r_tests.yml +++ b/.github/workflows/r_tests.yml @@ -105,25 +105,3 @@ jobs: if: steps.changes.outputs.r_package == 'true' run: | python3 ops/script/test_r_package.py --r=/usr/bin/R --task=doc - - build-r-docs: - name: Build docs for the R package - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=r-tests-build-docs - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/build-r-docs.sh - - name: Upload R doc - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket xgboost-docs \ - --prefix ${BRANCH_NAME}/${PR_COMMIT_SHA} --make-public \ - r-docs-${{ env.BRANCH_NAME }}.tar.bz2 diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py new file mode 100644 index 000000000000..a37ea759c0ab --- /dev/null +++ b/ops/pipeline/trigger-rtd.py @@ -0,0 +1,17 @@ +import os +import json + + +def build_proejct(): + event_path = os.environ["GITHUB_EVENT_PATH"] + print(f"Event path: {event_path}") + pr = json.loads(event_path).pull_request.number + + URL = f"https://readthedocs.org/api/v3/projects/xgboost/versions/{pr}/builds/" + HEADERS = {"Authorization": f"token {TOKEN}"} + response = requests.post(URL, headers=HEADERS) + print("response:", response) + assert response.status_code == 202 + response = response.json() + msg = json.dumps(json.loads(response), indent=2) + print("response:", response) From c55fef939eb5e1acc84ffbc6b347eea2200cb82f Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 00:44:28 +0800 Subject: [PATCH 24/42] Disable for now. --- .github/workflows/i386.yml | 27 --- .github/workflows/lint.yml | 101 ---------- .github/workflows/main.yml | 223 ---------------------- .github/workflows/misc.yml | 45 ----- .github/workflows/python_tests.yml | 64 ------- .github/workflows/python_wheels_macos.yml | 55 ------ .github/workflows/r_nold.yml | 41 ---- .github/workflows/r_tests.yml | 107 ----------- .github/workflows/scorecards.yml | 54 ------ .github/workflows/sycl_tests.yml | 48 ----- .github/workflows/windows.yml | 66 ------- 11 files changed, 831 deletions(-) delete mode 100644 .github/workflows/i386.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/misc.yml delete mode 100644 .github/workflows/python_tests.yml delete mode 100644 .github/workflows/python_wheels_macos.yml delete mode 100644 .github/workflows/r_nold.yml delete mode 100644 .github/workflows/r_tests.yml delete mode 100644 .github/workflows/scorecards.yml delete mode 100644 .github/workflows/sycl_tests.yml delete mode 100644 .github/workflows/windows.yml diff --git a/.github/workflows/i386.yml b/.github/workflows/i386.yml deleted file mode 100644 index 26ceaf758f3a..000000000000 --- a/.github/workflows/i386.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: XGBoost-i386-test - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build-32bit: - name: Build 32-bit - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=i386-build-32bit - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/test-cpp-i386.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 73636e7ce66d..000000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: XGBoost CI (Lint) - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - BRANCH_NAME: >- - ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} - -jobs: - clang-tidy: - name: Run clang-tidy - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=lint-clang-tidy - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/run-clang-tidy.sh - - python-mypy-lint: - runs-on: ubuntu-latest - name: Type and format checks for the Python package - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: dmlc/xgboost-devops/actions/miniforge-setup@main - with: - environment-name: python_lint - environment-file: ops/conda_env/python_lint.yml - - name: Run mypy - shell: bash -el {0} - run: | - python ops/script/lint_python.py --format=0 --type-check=1 --pylint=0 - - name: Run formatter - shell: bash -el {0} - run: | - python ops/script/lint_python.py --format=1 --type-check=0 --pylint=0 - - name: Run pylint - shell: bash -el {0} - run: | - python ops/script/lint_python.py --format=0 --type-check=0 --pylint=1 - - cpp-lint: - runs-on: ubuntu-latest - name: Code linting for C++ - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - architecture: 'x64' - - name: Install Python packages - run: | - python -m pip install wheel setuptools cmakelint cpplint==1.6.1 pylint - - name: Run lint - run: | - python3 ops/script/lint_cpp.py - bash ops/script/lint_cmake.sh - - lintr: - runs-on: ubuntu-latest - name: Run R linters on Ubuntu - env: - R_REMOTES_NO_ERRORS_FROM_WARNINGS: true - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: r-lib/actions/setup-r@v2 - with: - r-version: "release" - - name: Cache R packages - uses: actions/cache@v4 - with: - path: ${{ env.R_LIBS_USER }} - key: ${{ runner.os }}-r-release-7-${{ hashFiles('R-package/DESCRIPTION') }} - restore-keys: ${{ runner.os }}-r-release-7-${{ hashFiles('R-package/DESCRIPTION') }} - - name: Install dependencies - shell: Rscript {0} - run: | - source("./R-package/tests/helper_scripts/install_deps.R") - - name: Run lintr - run: | - MAKEFLAGS="-j$(nproc)" R CMD INSTALL R-package/ - Rscript ops/script/lint_r.R $(pwd) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 332606cece7b..74866f9c223c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,226 +35,3 @@ jobs: --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ --prefix cache/${{ github.run_id }}/build-cpu \ ./xgboost - - build-cpu-arm64: - name: Build CPU ARM64 + manylinux_2_28_aarch64 wheel - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-arm64-cpu - - tag=build-cpu-arm64 - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/build-cpu-arm64.sh - - name: Stash files - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/build-cpu-arm64 \ - ./xgboost python-package/dist/*.whl - - build-cuda: - name: Build CUDA + manylinux_2_28_x86_64 wheel - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=main-build-cuda - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: | - bash ops/pipeline/build-cuda.sh xgb-ci.gpu_build_rockylinux8 disable-rmm - - name: Stash files - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/build-cuda \ - build/testxgboost ./xgboost python-package/dist/*.whl - - build-cuda-with-rmm: - name: Build CUDA with RMM - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=main-build-cuda-with-rmm - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: | - bash ops/pipeline/build-cuda.sh xgb-ci.gpu_build_rockylinux8 enable-rmm - - name: Stash files - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/build-cuda-with-rmm \ - build/testxgboost - - build-cuda-with-rmm-dev: - name: Build CUDA with RMM (dev) - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=main-build-cuda-with-rmm-dev - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: | - bash ops/pipeline/build-cuda.sh \ - xgb-ci.gpu_build_rockylinux8_dev_ver enable-rmm - - build-manylinux2014: - name: Build manylinux2014_${{ matrix.arch }} wheel - runs-on: - - runs-on - - runner=${{ matrix.runner }} - - run-id=${{ github.run_id }} - - tag=main-build-manylinux2014-${{ matrix.arch }} - strategy: - fail-fast: false - matrix: - include: - - arch: aarch64 - runner: linux-arm64-cpu - - arch: x86_64 - runner: linux-amd64-cpu - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/build-manylinux2014.sh ${{ matrix.arch }} - - build-gpu-rpkg: - name: Build GPU-enabled R package - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=main-build-gpu-rpkg - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/build-gpu-rpkg.sh - - - test-cpp-gpu: - name: >- - Run Google Tests with GPUs - (Suite ${{ matrix.suite }}, Runner ${{ matrix.runner }}) - needs: [build-cuda, build-cuda-with-rmm] - runs-on: - - runs-on - - runner=${{ matrix.runner }} - - run-id=${{ github.run_id }} - - tag=main-test-cpp-gpu-${{ matrix.suite }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - suite: gpu - runner: linux-amd64-gpu - artifact_from: build-cuda - - suite: gpu-rmm - runner: linux-amd64-gpu - artifact_from: build-cuda-with-rmm - - suite: mgpu - runner: linux-amd64-mgpu - artifact_from: build-cuda - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - name: Unstash gtest - run: | - python3 ops/pipeline/manage-artifacts.py download \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/${{ matrix.artifact_from }} \ - --dest-dir build \ - testxgboost - chmod +x build/testxgboost - - run: bash ops/pipeline/test-cpp-gpu.sh ${{ matrix.suite }} - - test-python-wheel: - name: Run Python tests (${{ matrix.description }}) - needs: [build-cuda, build-cpu-arm64] - runs-on: - - runs-on - - runner=${{ matrix.runner }} - - run-id=${{ github.run_id }} - - tag=main-test-python-wheel-${{ matrix.description }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - description: single-gpu - image_repo: xgb-ci.gpu - suite: gpu - runner: linux-amd64-gpu - artifact_from: build-cuda - - description: multiple-gpu - image_repo: xgb-ci.gpu - suite: mgpu - runner: linux-amd64-mgpu - artifact_from: build-cuda - - description: cpu-amd64 - image_repo: xgb-ci.cpu - suite: cpu - runner: linux-amd64-cpu - artifact_from: build-cuda - - description: cpu-arm64 - image_repo: xgb-ci.aarch64 - suite: cpu-arm64 - runner: linux-arm64-cpu - artifact_from: build-cpu-arm64 - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - name: Unstash Python wheel - run: | - python3 ops/pipeline/manage-artifacts.py download \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/${{ matrix.artifact_from }} \ - --dest-dir wheelhouse \ - *.whl xgboost - mv -v wheelhouse/xgboost . - chmod +x ./xgboost - - name: Run Python tests, ${{ matrix.description }} - run: bash ops/pipeline/test-python-wheel.sh ${{ matrix.suite }} ${{ matrix.image_repo }} diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml deleted file mode 100644 index 54d0078a6164..000000000000 --- a/.github/workflows/misc.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: XGBoost CI (misc) - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - BRANCH_NAME: >- - ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} - -jobs: - gtest-cpu-nonomp: - name: Test Google C++ unittest (CPU Non-OMP) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Install system packages - run: | - sudo apt-get install -y --no-install-recommends ninja-build - - name: Build and test XGBoost - run: bash ops/pipeline/build-test-cpu-nonomp.sh - - c-api-demo: - name: Test installing XGBoost lib + building the C API demo - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: dmlc/xgboost-devops/actions/miniforge-setup@main - with: - environment-name: cpp_test - environment-file: ops/conda_env/cpp_test.yml - - name: Build and run C API demo with shared - run: bash ops/pipeline/test-c-api-demo.sh diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml deleted file mode 100644 index 180b1a855733..000000000000 --- a/.github/workflows/python_tests.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: XGBoost CI (Python tests) - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -defaults: - run: - shell: bash -l {0} - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - python-sdist-test: - runs-on: ${{ matrix.os }} - name: Test installing Python XGBoost from the source distribution (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - os: [macos-13, windows-latest, ubuntu-latest] - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: dmlc/xgboost-devops/actions/miniforge-setup@main - with: - environment-name: sdist_test - environment-file: ops/conda_env/sdist_test.yml - - name: Install extra package for MacOS - run: | - mamba install -c conda-forge llvm-openmp - if: matrix.os == 'macos-13' - - name: Build and install XGBoost - run: bash ops/pipeline/test-python-sdist.sh - - python-tests-on-macos: - name: Test XGBoost Python package on macos-13 - runs-on: macos-13 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: dmlc/xgboost-devops/actions/miniforge-setup@main - with: - environment-name: macos_cpu_test - environment-file: ops/conda_env/macos_cpu_test.yml - - run: bash ops/pipeline/test-python-macos.sh - - python-system-installation-on-ubuntu: - name: Test XGBoost Python package System Installation on Ubuntu - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - run: bash ops/pipeline/test-python-with-sysprefix.sh diff --git a/.github/workflows/python_wheels_macos.yml b/.github/workflows/python_wheels_macos.yml deleted file mode 100644 index 33eabbd09dca..000000000000 --- a/.github/workflows/python_wheels_macos.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Build Python wheels targeting MacOS - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -defaults: - run: - shell: bash -l {0} - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - BRANCH_NAME: >- - ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} - -jobs: - python-wheels-macos: - name: Build wheel for ${{ matrix.platform_id }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - os: macos-13 - platform_id: macosx_x86_64 - - os: macos-14 - platform_id: macosx_arm64 - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Set up homebrew - uses: Homebrew/actions/setup-homebrew@13341b4d5e459a98bbe0b122b12c11bf90518cc8 - - name: Install libomp - run: brew install libomp - - uses: dmlc/xgboost-devops/actions/miniforge-setup@main - with: - environment-name: minimal - environment-file: ops/conda_env/minimal.yml - - name: Build wheels - run: bash ops/pipeline/build-python-wheels-macos.sh ${{ matrix.platform_id }} ${{ github.sha }} - - name: Upload Python wheel - if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_') - run: | - python ops/pipeline/manage-artifacts.py upload \ - --s3-bucket xgboost-nightly-builds \ - --prefix ${{ env.BRANCH_NAME }}/${{ github.sha }} --make-public \ - wheelhouse/*.whl - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }} diff --git a/.github/workflows/r_nold.yml b/.github/workflows/r_nold.yml deleted file mode 100644 index da01f39f650b..000000000000 --- a/.github/workflows/r_nold.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Run expensive R tests with the help of rhub. Only triggered by a pull request review -# See discussion at https://github.com/dmlc/xgboost/pull/6378 - -name: XGBoost-R-noLD - -on: - pull_request_review_comment: - types: [created] - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - test-R-noLD: - if: github.event.comment.body == '/gha run r-nold-test' && contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association) - timeout-minutes: 120 - runs-on: ubuntu-latest - container: - image: rhub/debian-gcc-devel-nold - steps: - - name: Install git and system packages - shell: bash - run: | - apt update && apt install libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libglpk-dev libxml2-dev libharfbuzz-dev libfribidi-dev git -y - - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Install dependencies - shell: bash -l {0} - run: | - /tmp/R-devel/bin/Rscript -e "source('./R-package/tests/helper_scripts/install_deps.R')" - - name: Run R tests - shell: bash - run: | - cd R-package && \ - /tmp/R-devel/bin/R CMD INSTALL . && \ - /tmp/R-devel/bin/R -q -e "library(testthat); setwd('tests'); source('testthat.R')" diff --git a/.github/workflows/r_tests.yml b/.github/workflows/r_tests.yml deleted file mode 100644 index 7a7235cb2df9..000000000000 --- a/.github/workflows/r_tests.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: XGBoost-R-Tests - -on: [push, pull_request] - -env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - PR_COMMIT_SHA: ${{ github.event.pull_request.head.sha }} - BRANCH_NAME: >- - ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - - -jobs: - test-Rpkg: - runs-on: ${{ matrix.os }} - name: Test R on OS ${{ matrix.os }}, R ${{ matrix.r }}, Compiler ${{ matrix.compiler }}, Build ${{ matrix.build }} - strategy: - fail-fast: false - matrix: - include: - - os: windows-latest - r: release - compiler: mingw - build: autotools - - os: ubuntu-latest - r: release - compiler: none - build: cmake - env: - R_REMOTES_NO_ERRORS_FROM_WARNINGS: true - steps: - - name: Install system dependencies - run: | - sudo apt update - sudo apt install libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libglpk-dev libxml2-dev libharfbuzz-dev libfribidi-dev librsvg2-dev librsvg2-2 - if: matrix.os == 'ubuntu-latest' - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: r-lib/actions/setup-r@v2 - with: - r-version: ${{ matrix.r }} - - name: Cache R packages - uses: actions/cache@v4 - with: - path: ${{ env.R_LIBS_USER }} - key: ${{ runner.os }}-r-${{ matrix.r }}-8-${{ hashFiles('R-package/DESCRIPTION') }} - restore-keys: ${{ runner.os }}-r-${{ matrix.r }}-8-${{ hashFiles('R-package/DESCRIPTION') }} - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - architecture: 'x64' - - uses: r-lib/actions/setup-tinytex@v2 - - name: Install dependencies - shell: Rscript {0} - run: | - source("./R-package/tests/helper_scripts/install_deps.R") - - name: Test R - run: | - python ops/script/test_r_package.py --compiler='${{ matrix.compiler }}' --build-tool="${{ matrix.build }}" --task=check - if: matrix.compiler != 'none' - - name: Test R - run: | - python ops/script/test_r_package.py --build-tool="${{ matrix.build }}" --task=check - if: matrix.compiler == 'none' - - test-R-on-Debian: - name: Test R package on Debian - runs-on: ubuntu-latest - container: - image: rhub/debian-gcc-release - steps: - - name: Install system dependencies - run: | - # Must run before checkout to have the latest git installed. - # No need to add pandoc, the container has it figured out. - apt update && apt install libcurl4-openssl-dev libssl-dev libssh2-1-dev libgit2-dev libglpk-dev libxml2-dev libharfbuzz-dev libfribidi-dev git librsvg2-dev librsvg2-2 -y - - name: Trust git cloning project sources - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Install dependencies - shell: bash -l {0} - run: | - Rscript -e "source('./R-package/tests/helper_scripts/install_deps.R')" - - name: Test R - shell: bash -l {0} - run: | - python3 ops/script/test_r_package.py --r=/usr/bin/R --build-tool=autotools --task=check - - uses: dorny/paths-filter@v3 - id: changes - with: - filters: | - r_package: - - 'R-package/**' - - name: Run document check - if: steps.changes.outputs.r_package == 'true' - run: | - python3 ops/script/test_r_package.py --r=/usr/bin/R --task=doc diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml deleted file mode 100644 index f3837391b4fe..000000000000 --- a/.github/workflows/scorecards.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Scorecards supply-chain security -on: - # Only the default branch is supported. - branch_protection_rule: - schedule: - - cron: '17 2 * * 6' - push: - branches: [ "master" ] - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecards analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Used to receive a badge. - id-token: write - - steps: - - name: "Checkout code" - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 - with: - results_file: results.sarif - results_format: sarif - - # Publish the results for public repositories to enable scorecard badges. For more details, see - # https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories, `publish_results` will automatically be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard. - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@83a02f7883b12e0e4e1a146174f5e2292a01e601 # v2.16.4 - with: - sarif_file: results.sarif diff --git a/.github/workflows/sycl_tests.yml b/.github/workflows/sycl_tests.yml deleted file mode 100644 index 8efdc98d7fd9..000000000000 --- a/.github/workflows/sycl_tests.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: XGBoost CI (oneAPI) - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -defaults: - run: - shell: bash -l {0} - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - BRANCH_NAME: >- - ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} - -jobs: - gtest-cpu-sycl: - name: Test Google C++ unittest (CPU SYCL) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: dmlc/xgboost-devops/actions/miniforge-setup@main - with: - environment-name: linux_sycl_test - environment-file: ops/conda_env/linux_sycl_test.yml - - name: Run gtest - run: bash ops/pipeline/build-test-sycl.sh gtest - - python-sycl-tests-on-ubuntu: - name: Test XGBoost Python package with SYCL - runs-on: ubuntu-latest - timeout-minutes: 90 - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: dmlc/xgboost-devops/actions/miniforge-setup@main - with: - environment-name: linux_sycl_test - environment-file: ops/conda_env/linux_sycl_test.yml - - name: Test Python package - run: bash ops/pipeline/build-test-sycl.sh pytest diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index 53a1b5c0520b..000000000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: XGBoost CI (Windows) - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: powershell - -env: - BRANCH_NAME: >- - ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} - -jobs: - build-win64-gpu: - name: Build XGBoost for Windows with CUDA - runs-on: - - runs-on=${{ github.run_id }} - - runner=windows-cpu - - tag=windows-build-win64-gpu - steps: - - uses: actions/checkout@v4 - with: - submodules: "true" - - run: powershell ops/pipeline/build-win64-gpu.ps1 - - name: Stash files - shell: powershell - run: | - conda activate - python ops/pipeline/manage-artifacts.py upload ` - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} ` - --prefix cache/${{ github.run_id }}/build-win64-gpu ` - build/testxgboost.exe xgboost.exe ` - (Get-ChildItem python-package/dist/*.whl | Select-Object -Expand FullName) - - test-win64-gpu: - name: Test XGBoost on Windows - needs: build-win64-gpu - runs-on: - - runs-on=${{ github.run_id }} - - runner=windows-gpu - - tag=windows-test-win64-gpu - steps: - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Unstash files - shell: powershell - run: | - conda activate - python ops/pipeline/manage-artifacts.py download ` - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} ` - --prefix cache/${{ github.run_id }}/build-win64-gpu ` - --dest-dir build ` - *.whl testxgboost.exe xgboost.exe - Move-Item -Path build/xgboost.exe -Destination . - New-Item -ItemType Directory -Path python-package/dist/ -Force - Move-Item -Path (Get-ChildItem build/*.whl | Select-Object -Expand FullName) ` - -Destination python-package/dist/ - - run: powershell ops/pipeline/test-win64-gpu.ps1 From 6df263d3cc0719f4e893d5e0f542916b642ebc53 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 00:48:32 +0800 Subject: [PATCH 25/42] Fix. --- .github/workflows/doc.yml | 4 + .github/workflows/freebsd.yml | 29 ---- .github/workflows/jvm_tests.yml | 276 -------------------------------- 3 files changed, 4 insertions(+), 305 deletions(-) delete mode 100644 .github/workflows/freebsd.yml delete mode 100644 .github/workflows/jvm_tests.yml diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 0993491ce923..8aa229a6b55e 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -2,6 +2,10 @@ name: XGBoost-docs on: [push, pull_request] +env: + BRANCH_NAME: >- + ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} + jobs: build-r-docs: diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml deleted file mode 100644 index 26e8fa34c119..000000000000 --- a/.github/workflows/freebsd.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: FreeBSD - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 20 - name: A job to run test in FreeBSD - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Test in FreeBSD - id: test - uses: vmactions/freebsd-vm@v1 - with: - usesh: true - prepare: | - pkg install -y cmake git ninja googletest bash - run: | - bash ops/pipeline/test-freebsd.sh diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml deleted file mode 100644 index d88381146fe0..000000000000 --- a/.github/workflows/jvm_tests.yml +++ /dev/null @@ -1,276 +0,0 @@ -name: XGBoost CI (JVM packages) - -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - PR_COMMIT_SHA: ${{ github.event.pull_request.head.sha }} - BRANCH_NAME: >- - ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} - -jobs: - build-jvm-manylinux2014: - name: >- - Build libxgboost4j.so targeting glibc 2.17 - (arch ${{ matrix.arch }}, runner ${{ matrix.runner }}) - runs-on: - - runs-on - - runner=${{ matrix.runner }} - - run-id=${{ github.run_id }} - - tag=jvm-tests-build-jvm-manylinux2014-${{ matrix.arch }} - strategy: - fail-fast: false - matrix: - include: - - arch: aarch64 - runner: linux-arm64-cpu - - arch: x86_64 - runner: linux-amd64-cpu - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/build-jvm-manylinux2014.sh ${{ matrix.arch }} - - build-jvm-gpu: - name: Build libxgboost4j.so with CUDA - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=jvm-tests-build-jvm-gpu - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/build-jvm-gpu.sh - - name: Stash files - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/build-jvm-gpu \ - lib/libxgboost4j.so - - build-jvm-mac: - name: "Build libxgboost4j.dylib for ${{ matrix.description }}" - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - description: "MacOS (Apple Silicon)" - script: ops/pipeline/build-jvm-macos-apple-silicon.sh - libname: libxgboost4j_m1.dylib - runner: macos-14 - - description: "MacOS (Intel)" - script: ops/pipeline/build-jvm-macos-intel.sh - libname: libxgboost4j_intel.dylib - runner: macos-13 - steps: - - uses: actions/checkout@v4 - with: - submodules: "true" - - run: bash ${{ matrix.script }} - - name: Upload libxgboost4j.dylib - if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_') - run: | - mv -v lib/libxgboost4j.dylib ${{ matrix.libname }} - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket xgboost-nightly-builds \ - --prefix ${{ env.BRANCH_NAME }}/${{ github.sha }} --make-public \ - ${{ matrix.libname }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }} - - build-jvm-docs: - name: Build docs for JVM packages - needs: [build-jvm-gpu] - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=jvm-tests-build-jvm-docs - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - name: Unstash files - run: | - python3 ops/pipeline/manage-artifacts.py download \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/build-jvm-gpu \ - --dest-dir lib \ - libxgboost4j.so - - run: bash ops/pipeline/build-jvm-doc.sh - - name: Upload JVM doc - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket xgboost-docs \ - --prefix ${BRANCH_NAME}/${PR_COMMIT_SHA} --make-public \ - jvm-packages/${{ env.BRANCH_NAME }}.tar.bz2 - - build-test-jvm-packages: - name: Build and test JVM packages (Linux, Scala ${{ matrix.scala_version }}) - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=jvm-tests-build-test-jvm-packages-scala${{ matrix.scala_version }} - strategy: - fail-fast: false - matrix: - scala_version: ["2.12", "2.13"] - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - name: Build and test JVM packages (Scala ${{ matrix.scala_version }}) - run: bash ops/pipeline/build-test-jvm-packages.sh - env: - SCALA_VERSION: ${{ matrix.scala_version }} - - name: Stash files - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/build-test-jvm-packages \ - lib/libxgboost4j.so - if: matrix.scala_version == '2.13' - - build-test-jvm-packages-other-os: - name: Build and test JVM packages (${{ matrix.os }}) - timeout-minutes: 30 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [windows-latest, macos-13] - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '8' - - uses: dmlc/xgboost-devops/actions/miniforge-setup@main - with: - environment-name: minimal - environment-file: ops/conda_env/minimal.yml - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }} - restore-keys: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }} - - name: Test XGBoost4J (Core) on macos - if: matrix.os == 'macos-13' - run: | - cd jvm-packages - mvn test -B -pl :xgboost4j_2.12 -Duse.openmp=OFF - - name: Test XGBoost4J (Core) on windows - if: matrix.os == 'windows-latest' - run: | - cd jvm-packages - mvn test -B -pl :xgboost4j_2.12 - - name: Publish artifact xgboost4j.dll to S3 - run: | - python ops/pipeline/manage-artifacts.py upload ` - --s3-bucket xgboost-nightly-builds ` - --prefix ${{ env.BRANCH_NAME }}/${{ github.sha }} --make-public ` - lib/xgboost4j.dll - if: | - (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')) && - matrix.os == 'windows-latest' - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }} - - test-jvm-packages-gpu: - name: Test JVM packages with CUDA (Scala ${{ matrix.scala_version }}) - needs: [build-jvm-gpu] - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-mgpu - - tag=jvm-tests-test-jvm-packages-gpu-scala${{ matrix.scala_version }} - strategy: - fail-fast: false - matrix: - scala_version: ["2.12", "2.13"] - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - name: Unstash files - run: | - python3 ops/pipeline/manage-artifacts.py download \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/build-jvm-gpu \ - --dest-dir lib \ - libxgboost4j.so - - run: bash ops/pipeline/test-jvm-gpu.sh - env: - SCALA_VERSION: ${{ matrix.scala_version }} - - deploy-jvm-packages: - name: Deploy JVM packages to S3 (${{ matrix.variant.name }}) - needs: [build-jvm-gpu, build-test-jvm-packages, test-jvm-packages-gpu] - runs-on: - - runs-on - - runner=linux-amd64-cpu - - run-id=${{ github.run_id }} - - tag=jvm-tests-deploy-jvm-packages-${{ matrix.variant.name }}-scala${{ matrix.scala_version }} - strategy: - fail-fast: false - matrix: - variant: - - name: cpu - image_repo: xgb-ci.jvm - artifact_from: build-test-jvm-packages - - name: gpu - image_repo: xgb-ci.jvm_gpu_build - artifact_from: build-jvm-gpu - scala_version: ['2.12', '2.13'] - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - name: Unstash files - run: | - python3 ops/pipeline/manage-artifacts.py download \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/${{ matrix.variant.artifact_from }} \ - --dest-dir lib \ - libxgboost4j.so - ls -lh lib/libxgboost4j.so - - name: Deploy JVM packages to S3 - run: | - bash ops/pipeline/deploy-jvm-packages.sh ${{ matrix.variant.name }} \ - ${{ matrix.variant.image_repo }} ${{ matrix.scala_version }} From cfdb4320b81b239b1cbcb0b1dcf66f90fbde6c1c Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 01:05:06 +0800 Subject: [PATCH 26/42] setup. --- .github/workflows/doc.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 8aa229a6b55e..4916a57e0bca 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -35,5 +35,12 @@ jobs: name: Trigger Read The Docs build. runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + with: + submodules: 'true' + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + architecture: 'x64' - run: | python3 ops/pipeline/trigger-rtd.py From 383ffbf78533abad28a410451e68f9075fbd51a5 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 01:40:15 +0800 Subject: [PATCH 27/42] test. --- .github/workflows/doc.yml | 2 ++ ops/pipeline/trigger-rtd.py | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 4916a57e0bca..0faa4ff71957 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -43,4 +43,6 @@ jobs: python-version: "3.10" architecture: 'x64' - run: | + pip3 install requests + # fixme: add token here. python3 ops/pipeline/trigger-rtd.py diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index a37ea759c0ab..987455a87202 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -1,17 +1,28 @@ -import os +import argparse import json +import os + +import requests -def build_proejct(): +def trigger_build(token: str) -> None: event_path = os.environ["GITHUB_EVENT_PATH"] print(f"Event path: {event_path}") pr = json.loads(event_path).pull_request.number URL = f"https://readthedocs.org/api/v3/projects/xgboost/versions/{pr}/builds/" - HEADERS = {"Authorization": f"token {TOKEN}"} + HEADERS = {"Authorization": f"token {token}"} response = requests.post(URL, headers=HEADERS) print("response:", response) + # 202 means the build is successfully triggerred. assert response.status_code == 202 response = response.json() msg = json.dumps(json.loads(response), indent=2) print("response:", response) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--token", type=str, default=None) # fixme + args = parser.parse_args() + trigger_build(args.token) From 9b7a3f1e5e7c2de11a03b21931a7190cb861f15d Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 16:08:47 +0800 Subject: [PATCH 28/42] auth. --- .github/workflows/doc.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 0faa4ff71957..a2012d52d015 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -44,5 +44,9 @@ jobs: architecture: 'x64' - run: | pip3 install requests + - name: Trigger RTD + run: | # fixme: add token here. - python3 ops/pipeline/trigger-rtd.py + python3 ops/pipeline/trigger-rtd.py --token=${RTD_AUTH_TOKEN} + env: + RTD_AUTH_TOKEN: ${{ secrets.RTD_AUTH_TOKEN }} From e484df0877171c68d7ee0e12be218fd245be27f3 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 16:23:11 +0800 Subject: [PATCH 29/42] Move the notes. --- doc/conf.py | 47 ++++--------------------------------- doc/contrib/docs.rst | 34 +++++++++++++++++++++++++++ ops/pipeline/trigger-rtd.py | 10 +++++++- 3 files changed, 48 insertions(+), 43 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index cf47022ac6fa..24f53d4c05c7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,21 +1,8 @@ -# -*- coding: utf-8 -*- -# -# documentation build configuration file, created by -# sphinx-quickstart on Thu Jul 23 19:40:08 2015. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. -# -# Envs: -# - READTHEDOCS: Read the docs flag, fetch the R and JVM documents. -# - XGBOOST_R_DOCS: Local path for pre-build R document, used for development. -# - XGBOOST_JVM_DOCS: Local path for pre-build JVM document, used for development. +"""Sphinx configuration. + +See `doc/contrib/docs.rst `__ +for more info. +""" import os import shutil @@ -43,30 +30,6 @@ release = xgboost.__version__ -# Witchcraft alert: -# -# Both the jvm and the R document is built with an independent CI pipeline and fetched -# during document build. -# -# The fetched artifacts are stored in xgboost/doc/tmp/jvm_docs and -# xgboost/doc/tmp/r_docs respectively. For the R package, there's a dummy index file in -# xgboost/doc/R-package/r_docs . Jvm doc is similar. As for the C doc, it's generated -# using doxygen and processed by breathe during build and there's no independent CI -# pipeline as it's relatively cheap. The generated xml files are stored in -# xgboost/doc/tmp/dev . -# -# The xgboost/doc/tmp is part of the `html_extra_path` sphinx configuration, which -# somehow makes sphinx to copy the extracted html files to the build directory. -# -# There's very limited dependency management in the readthedocs builder, to avoid -# building everything from source using the RTD worker (like packages from CRAN or -# XGBoost itself), we offload to CI workers as much as possible. -# -# However, since there's no way to synchronize the RTD worker with other GitHub workers, -# to see the latest document for a PR branch, we need to re-run RTD manually after -# related doc are uploaded to the S3 bucket. - - # Document is uploaded to here by the CI builder. S3_BUCKET = "https://xgboost-docs.s3.us-west-2.amazonaws.com" diff --git a/doc/contrib/docs.rst b/doc/contrib/docs.rst index df2b2692a28a..cb637dcd16a2 100644 --- a/doc/contrib/docs.rst +++ b/doc/contrib/docs.rst @@ -81,6 +81,40 @@ Build the Python Docs using pip and Conda Build finished. The HTML pages are in _build/html. +************* +Read The Docs +************* + +`Read the Docs `__ (RTD for short) is an online document hosting +service and hosts the `XGBoost document site +`__. The document builder used by RTD is +relatively lightweight but some of the packages like the R binding require a compiled +XGBoost along with all the optional dependencies to render the document. As a result, both +the jvm and the R document is built with an independent CI pipeline and fetched during +online document build. + +The sphinx configuration file ``xgboost/doc/conf.py`` acts as the fetcher. During build, +the fetched artifacts are stored in ``xgboost/doc/tmp/jvm_docs`` and +``xgboost/doc/tmp/r_docs`` respectively. For the R package, there's a dummy index file in +``xgboost/doc/R-package/r_docs`` . Jvm doc is similar. As for the C doc, it's generated +using doxygen and processed by breathe during build and there's no independent CI pipeline +as it's relatively cheap. The generated xml files are stored in ``xgboost/doc/tmp/dev`` . + +The ``xgboost/doc/tmp`` is part of the ``html_extra_path`` sphinx configuration, which +somehow makes sphinx to copy the extracted html files to the build directory. Following is +a list of environment variables used by the fetcher ``conf.py``: + + - READTHEDOCS: Read the docs flag, fetch the R and JVM documents when set to ``True`` + (case sensitive). + - XGBOOST_R_DOCS: Local path for pre-build R document, used for development. + - XGBOOST_JVM_DOCS: Local path for pre-build JVM document, used for development. + +As of writing, RTD doesn't provide any facility to be embedded as a GitHub action but we +need a way to specify the dependency between the CI pipeline and the document build in +order to fetch the correct artifact. The workaround is to disable to default webhook from +RTD and use an extra GA step to notify RTD using its `REST API +`__. + ******** Examples ******** diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index 987455a87202..ae2c1dd0129c 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -1,3 +1,10 @@ +"""Helper script for triggering Read the docs build. + +See `doc/contrib/docs.rst `__ +for more info. + +""" + import argparse import json import os @@ -6,6 +13,7 @@ def trigger_build(token: str) -> None: + """Trigger RTD build.""" event_path = os.environ["GITHUB_EVENT_PATH"] print(f"Event path: {event_path}") pr = json.loads(event_path).pull_request.number @@ -23,6 +31,6 @@ def trigger_build(token: str) -> None: if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--token", type=str, default=None) # fixme + parser.add_argument("--token", type=str, required=True) # fixme args = parser.parse_args() trigger_build(args.token) From a22faa5d179704d12d9d05cfd06d75ee65ca32c8 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 16:49:13 +0800 Subject: [PATCH 30/42] ep. --- .github/workflows/doc.yml | 1 - .github/workflows/main.yml | 9 --------- ops/pipeline/trigger-rtd.py | 5 +++-- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index a2012d52d015..b85dfde6130d 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -46,7 +46,6 @@ jobs: pip3 install requests - name: Trigger RTD run: | - # fixme: add token here. python3 ops/pipeline/trigger-rtd.py --token=${RTD_AUTH_TOKEN} env: RTD_AUTH_TOKEN: ${{ secrets.RTD_AUTH_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 74866f9c223c..5164c4109d4b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,12 +26,3 @@ jobs: - uses: actions/checkout@v4 with: submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/build-cpu.sh - - name: Stash CLI executable - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket ${{ env.RUNS_ON_S3_BUCKET_CACHE }} \ - --prefix cache/${{ github.run_id }}/build-cpu \ - ./xgboost diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index ae2c1dd0129c..da1921bab09f 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -16,7 +16,8 @@ def trigger_build(token: str) -> None: """Trigger RTD build.""" event_path = os.environ["GITHUB_EVENT_PATH"] print(f"Event path: {event_path}") - pr = json.loads(event_path).pull_request.number + with open(event_path, "r") as fd: + pr = json.load(fd)["pull_request"]["number"] URL = f"https://readthedocs.org/api/v3/projects/xgboost/versions/{pr}/builds/" HEADERS = {"Authorization": f"token {token}"} @@ -31,6 +32,6 @@ def trigger_build(token: str) -> None: if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--token", type=str, required=True) # fixme + parser.add_argument("--token", type=str, required=True) args = parser.parse_args() trigger_build(args.token) From fa9b91e34fa60fcd227d7d943180d5edb79ef71b Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 17:09:46 +0800 Subject: [PATCH 31/42] debug. --- ops/pipeline/trigger-rtd.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index da1921bab09f..5b925fa8dbb9 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -17,9 +17,12 @@ def trigger_build(token: str) -> None: event_path = os.environ["GITHUB_EVENT_PATH"] print(f"Event path: {event_path}") with open(event_path, "r") as fd: - pr = json.load(fd)["pull_request"]["number"] + event = json.load(fd) + print(f"Event: {event}") + pr = event["pull_request"]["number"] URL = f"https://readthedocs.org/api/v3/projects/xgboost/versions/{pr}/builds/" + print(f"URL: {URL}") HEADERS = {"Authorization": f"token {token}"} response = requests.post(URL, headers=HEADERS) print("response:", response) From 27a80ab12d8991d9eccd86346178b5d6c59edaa2 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 18:09:50 +0800 Subject: [PATCH 32/42] debug. --- .github/workflows/doc.yml | 44 ++++++++++++++++++------------------- ops/pipeline/trigger-rtd.py | 8 ++++--- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index b85dfde6130d..c0216edd647a 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -8,30 +8,30 @@ env: jobs: - build-r-docs: - name: Build docs for the R package - runs-on: - - runs-on=${{ github.run_id }} - - runner=linux-amd64-cpu - - tag=r-tests-build-docs - steps: - # Restart Docker daemon so that it recognizes the ephemeral disks - - run: sudo systemctl restart docker - - uses: actions/checkout@v4 - with: - submodules: "true" - - name: Log into Docker registry (AWS ECR) - run: bash ops/pipeline/login-docker-registry.sh - - run: bash ops/pipeline/build-r-docs.sh - - name: Upload R doc - run: | - python3 ops/pipeline/manage-artifacts.py upload \ - --s3-bucket xgboost-docs \ - --prefix ${BRANCH_NAME}/${PR_COMMIT_SHA} --make-public \ - r-docs-${{ env.BRANCH_NAME }}.tar.bz2 + # build-r-docs: + # name: Build docs for the R package + # runs-on: + # - runs-on=${{ github.run_id }} + # - runner=linux-amd64-cpu + # - tag=r-tests-build-docs + # steps: + # # Restart Docker daemon so that it recognizes the ephemeral disks + # - run: sudo systemctl restart docker + # - uses: actions/checkout@v4 + # with: + # submodules: "true" + # - name: Log into Docker registry (AWS ECR) + # run: bash ops/pipeline/login-docker-registry.sh + # - run: bash ops/pipeline/build-r-docs.sh + # - name: Upload R doc + # run: | + # python3 ops/pipeline/manage-artifacts.py upload \ + # --s3-bucket xgboost-docs \ + # --prefix ${BRANCH_NAME}/${PR_COMMIT_SHA} --make-public \ + # r-docs-${{ env.BRANCH_NAME }}.tar.bz2 trigger-rtd-build: - needs: build-r-docs + # needs: build-r-docs name: Trigger Read The Docs build. runs-on: ubuntu-latest steps: diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index 5b925fa8dbb9..c2b948615021 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -14,6 +14,8 @@ def trigger_build(token: str) -> None: """Trigger RTD build.""" + # we should be really careful to not log (expose) the token in the CI. + assert token is not None event_path = os.environ["GITHUB_EVENT_PATH"] print(f"Event path: {event_path}") with open(event_path, "r") as fd: @@ -26,10 +28,10 @@ def trigger_build(token: str) -> None: HEADERS = {"Authorization": f"token {token}"} response = requests.post(URL, headers=HEADERS) print("response:", response) - # 202 means the build is successfully triggerred. + # 202 means the build is successfully triggered. assert response.status_code == 202 response = response.json() - msg = json.dumps(json.loads(response), indent=2) + msg = json.dumps(response, indent=2) print("response:", response) @@ -37,4 +39,4 @@ def trigger_build(token: str) -> None: parser = argparse.ArgumentParser() parser.add_argument("--token", type=str, required=True) args = parser.parse_args() - trigger_build(args.token) + trigger_build(args.token.strip()) From 1c8f391ff0e00bb19d857d483d1c1a26f1509fb0 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 18:18:40 +0800 Subject: [PATCH 33/42] debug. --- ops/pipeline/trigger-rtd.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index c2b948615021..c384b908b4b3 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -16,6 +16,9 @@ def trigger_build(token: str) -> None: """Trigger RTD build.""" # we should be really careful to not log (expose) the token in the CI. assert token is not None + if not isinstance(token, str) or len(token) != 40: + raise ValueError("Invalid token") + event_path = os.environ["GITHUB_EVENT_PATH"] print(f"Event path: {event_path}") with open(event_path, "r") as fd: @@ -37,6 +40,8 @@ def trigger_build(token: str) -> None: if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--token", type=str, required=True) + parser.add_argument( + "--token", type=str, required=True, help="API token for Read the docs." + ) args = parser.parse_args() trigger_build(args.token.strip()) From 257ed00fd018a2e1c2d55a4b38581f2590d026ff Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 18:20:58 +0800 Subject: [PATCH 34/42] debug. --- .github/workflows/doc.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index c0216edd647a..67f3c27b8309 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -3,6 +3,7 @@ name: XGBoost-docs on: [push, pull_request] env: + RTD_AUTH_TOKEN: ${{ secrets.RTD_AUTH_TOKEN }} BRANCH_NAME: >- ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} @@ -46,6 +47,5 @@ jobs: pip3 install requests - name: Trigger RTD run: | + echo "branch name: ${BRANCH_NAME}" python3 ops/pipeline/trigger-rtd.py --token=${RTD_AUTH_TOKEN} - env: - RTD_AUTH_TOKEN: ${{ secrets.RTD_AUTH_TOKEN }} From 719f88b8367b090cb11a5d0d55a77a2ba82969cb Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 18:26:28 +0800 Subject: [PATCH 35/42] debug. --- .github/workflows/doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 67f3c27b8309..88af5e258ec0 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -48,4 +48,4 @@ jobs: - name: Trigger RTD run: | echo "branch name: ${BRANCH_NAME}" - python3 ops/pipeline/trigger-rtd.py --token=${RTD_AUTH_TOKEN} + python3 ops/pipeline/trigger-rtd.py --token=${{ env.RTD_AUTH_TOKEN }} From 2b6b8ad8d8baef45533d81d0ba3716da53c7c5aa Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 18:29:56 +0800 Subject: [PATCH 36/42] Print revoked token. --- ops/pipeline/trigger-rtd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index c384b908b4b3..dc2eed21b36b 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -17,7 +17,7 @@ def trigger_build(token: str) -> None: # we should be really careful to not log (expose) the token in the CI. assert token is not None if not isinstance(token, str) or len(token) != 40: - raise ValueError("Invalid token") + raise ValueError(f"Invalid token: {token}") event_path = os.environ["GITHUB_EVENT_PATH"] print(f"Event path: {event_path}") From fdadf8ae55b26b9e3db8f112217dae6b232373b7 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 18:32:46 +0800 Subject: [PATCH 37/42] debug. --- .github/workflows/doc.yml | 1 + ops/pipeline/trigger-rtd.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 88af5e258ec0..9c3a77b04748 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -48,4 +48,5 @@ jobs: - name: Trigger RTD run: | echo "branch name: ${BRANCH_NAME}" + echo "token: ${RTD_AUTH_TOKEN}" python3 ops/pipeline/trigger-rtd.py --token=${{ env.RTD_AUTH_TOKEN }} diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index dc2eed21b36b..81ac1e8110a5 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -17,6 +17,7 @@ def trigger_build(token: str) -> None: # we should be really careful to not log (expose) the token in the CI. assert token is not None if not isinstance(token, str) or len(token) != 40: + # fixme: remove this once we have a valid token raise ValueError(f"Invalid token: {token}") event_path = os.environ["GITHUB_EVENT_PATH"] From 00e7b47623b510402c9275f2ff07908507a28f0d Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 18:39:16 +0800 Subject: [PATCH 38/42] use env. --- .github/workflows/doc.yml | 5 +++-- ops/pipeline/trigger-rtd.py | 20 ++++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 9c3a77b04748..0e7bb27cbfd3 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -3,7 +3,6 @@ name: XGBoost-docs on: [push, pull_request] env: - RTD_AUTH_TOKEN: ${{ secrets.RTD_AUTH_TOKEN }} BRANCH_NAME: >- ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} @@ -46,7 +45,9 @@ jobs: - run: | pip3 install requests - name: Trigger RTD + env: + RTD_AUTH_TOKEN: ${{ secrets.RTD_AUTH_TOKEN }} run: | echo "branch name: ${BRANCH_NAME}" echo "token: ${RTD_AUTH_TOKEN}" - python3 ops/pipeline/trigger-rtd.py --token=${{ env.RTD_AUTH_TOKEN }} + python3 ops/pipeline/trigger-rtd.py --token=${RTD_AUTH_TOKEN} diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index 81ac1e8110a5..da7f81216782 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -5,7 +5,6 @@ """ -import argparse import json import os @@ -14,11 +13,6 @@ def trigger_build(token: str) -> None: """Trigger RTD build.""" - # we should be really careful to not log (expose) the token in the CI. - assert token is not None - if not isinstance(token, str) or len(token) != 40: - # fixme: remove this once we have a valid token - raise ValueError(f"Invalid token: {token}") event_path = os.environ["GITHUB_EVENT_PATH"] print(f"Event path: {event_path}") @@ -40,9 +34,11 @@ def trigger_build(token: str) -> None: if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--token", type=str, required=True, help="API token for Read the docs." - ) - args = parser.parse_args() - trigger_build(args.token.strip()) + token = os.getenv("RTD_AUTH_TOKEN") + # We should be really careful to not log (expose) the token in the CI. + assert token is not None + if not isinstance(token, str) or len(token) != 40: + # fixme: remove this once we have a valid token + raise ValueError(f"Invalid token: {token}") + + trigger_build(token) From a0cac783b260de51b26c5f30380b2b9562c242e4 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 18:49:32 +0800 Subject: [PATCH 39/42] redact? --- ops/pipeline/trigger-rtd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index da7f81216782..301bd83f6caf 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -39,6 +39,6 @@ def trigger_build(token: str) -> None: assert token is not None if not isinstance(token, str) or len(token) != 40: # fixme: remove this once we have a valid token - raise ValueError(f"Invalid token: {token}") + raise ValueError(f"Invalid token: {len(token)}") trigger_build(token) From f04f708c1e537ba3eb91fd58c9841efbb0ebb764 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 19:09:08 +0800 Subject: [PATCH 40/42] debug. --- .github/workflows/doc.yml | 6 +++--- ops/pipeline/trigger-rtd.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 0e7bb27cbfd3..7b4254389a45 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -46,8 +46,8 @@ jobs: pip3 install requests - name: Trigger RTD env: - RTD_AUTH_TOKEN: ${{ secrets.RTD_AUTH_TOKEN }} + RTD_AUTH: ${{ secrets.RTD_AUTH_TOKEN }} run: | echo "branch name: ${BRANCH_NAME}" - echo "token: ${RTD_AUTH_TOKEN}" - python3 ops/pipeline/trigger-rtd.py --token=${RTD_AUTH_TOKEN} + echo "token: ${RTD_AUTH}" + python3 ops/pipeline/trigger-rtd.py diff --git a/ops/pipeline/trigger-rtd.py b/ops/pipeline/trigger-rtd.py index 301bd83f6caf..fe2fe8feacac 100644 --- a/ops/pipeline/trigger-rtd.py +++ b/ops/pipeline/trigger-rtd.py @@ -34,7 +34,7 @@ def trigger_build(token: str) -> None: if __name__ == "__main__": - token = os.getenv("RTD_AUTH_TOKEN") + token = os.getenv("RTD_AUTH") # We should be really careful to not log (expose) the token in the CI. assert token is not None if not isinstance(token, str) or len(token) != 40: From 907e28e101c126c60222bd4c807a8dffe3e694f4 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 19:16:09 +0800 Subject: [PATCH 41/42] test another one. --- .github/workflows/doc.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 7b4254389a45..488bd5fc9f7b 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -2,6 +2,13 @@ name: XGBoost-docs on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: BRANCH_NAME: >- ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} @@ -47,7 +54,9 @@ jobs: - name: Trigger RTD env: RTD_AUTH: ${{ secrets.RTD_AUTH_TOKEN }} + TEST_SEC: $${ secrets.TEST_SECRET }} run: | echo "branch name: ${BRANCH_NAME}" - echo "token: ${RTD_AUTH}" + echo "token: ${#RTD_AUTH}" + echo "test: ${#TEST_SEC}" python3 ops/pipeline/trigger-rtd.py From 98c94a70d75836f04c959056775357087dc04486 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Fri, 7 Feb 2025 19:17:31 +0800 Subject: [PATCH 42/42] Fix. --- .github/workflows/doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 488bd5fc9f7b..36daf9bbf04b 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -54,7 +54,7 @@ jobs: - name: Trigger RTD env: RTD_AUTH: ${{ secrets.RTD_AUTH_TOKEN }} - TEST_SEC: $${ secrets.TEST_SECRET }} + TEST_SEC: ${{ secrets.TEST_SECRET }} run: | echo "branch name: ${BRANCH_NAME}" echo "token: ${#RTD_AUTH}"