Skip to content

Commit

Permalink
Merge branch 'ray-py312' of https://github.com/ray-project/ray into p…
Browse files Browse the repository at this point in the history
…y312/ml-fixes
  • Loading branch information
justinvyu committed Jul 22, 2024
2 parents 7d3b152 + fcd772d commit 9a840a1
Show file tree
Hide file tree
Showing 25 changed files with 555 additions and 774 deletions.
6 changes: 2 additions & 4 deletions .buildkite/others.rayci.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
group: others
depends_on:
- oss-ci-base_build
- forge
steps:
#build
Expand All @@ -18,10 +17,9 @@ steps:
- ./ci/ci.sh compile_pip_dependencies
- cp -f ./python/requirements_compiled.txt /artifact-mount/
soft_fail: true
job_env: oss-ci-base_build-py3.12
job_env: oss-ci-base_test-py3.12
depends_on:
- forge
- oss-ci-base_build-multipy
- oss-ci-base_test-multipy

# test
- label: doc tests
Expand Down
10 changes: 8 additions & 2 deletions ci/docker/base.build.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,11 @@ RUN apt-get install -y -qq maven openjdk-8-jre openjdk-8-jdk

COPY . .

# init also calls install-dependencies.sh (again)
RUN BUILD=1 ./ci/ci.sh init
RUN <<EOF
#!/bin/bash -i

set -euo pipefail

BUILD=1 ./ci/ci.sh init

EOF
1 change: 1 addition & 0 deletions ci/docker/base.gpu.py39.wanda.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ srcs:
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/env/install-bazel.sh
- ci/env/install-miniconda.sh
- ci/pipeline/determine_tests_to_run.py
- ci/suppress_output
- python/requirements.txt
Expand Down
1 change: 1 addition & 0 deletions ci/docker/base.gpu.wanda.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ srcs:
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/env/install-bazel.sh
- ci/env/install-miniconda.sh
- ci/pipeline/determine_tests_to_run.py
- ci/suppress_output
- python/requirements.txt
Expand Down
3 changes: 1 addition & 2 deletions ci/docker/base.ml.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,12 @@ FROM $DOCKER_IMAGE_BASE_TEST

COPY . .

RUN BUILD=1 ./ci/ci.sh init

RUN <<EOF
#!/bin/bash -i

set -e

BUILD=1 ./ci/ci.sh init
RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh

pip uninstall -y ray
Expand Down
1 change: 1 addition & 0 deletions ci/docker/base.test.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,5 @@ WORKDIR /ray
# Below should be re-run each time
COPY . .

RUN ./ci/env/install-miniconda.sh
RUN ./ci/env/install-bazel.sh
2 changes: 2 additions & 0 deletions ci/docker/base.test.aarch64.wanda.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ froms: ["ubuntu:focal"]
dockerfile: ci/docker/base.test.Dockerfile
srcs:
- ci/env/install-bazel.sh
- ci/env/install-miniconda.sh
- ci/suppress_output
- .bazelversion
build_args:
- BUILDKITE_BAZEL_CACHE_URL
Expand Down
2 changes: 2 additions & 0 deletions ci/docker/base.test.py39.wanda.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ froms: ["ubuntu:focal"]
dockerfile: ci/docker/base.test.Dockerfile
srcs:
- ci/env/install-bazel.sh
- ci/env/install-miniconda.sh
- ci/suppress_output
- .bazelversion
build_args:
- BUILDKITE_BAZEL_CACHE_URL
Expand Down
2 changes: 2 additions & 0 deletions ci/docker/base.test.wanda.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ froms: ["ubuntu:focal"]
dockerfile: ci/docker/base.test.Dockerfile
srcs:
- ci/env/install-bazel.sh
- ci/env/install-miniconda.sh
- ci/suppress_output
- .bazelversion
build_args:
- BUILDKITE_BAZEL_CACHE_URL
Expand Down
108 changes: 3 additions & 105 deletions ci/env/install-dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ set -euxo pipefail
SCRIPT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
WORKSPACE_DIR="${SCRIPT_DIR}/../.."

# importing install_miniconda function
source "${SCRIPT_DIR}/install-miniconda.sh"

pkg_install_helper() {
case "${OSTYPE}" in
darwin*)
Expand Down Expand Up @@ -65,111 +68,6 @@ install_base() {
esac
}

install_miniconda() {
if [ "${OSTYPE}" = msys ]; then
# Windows is on GitHub Actions, whose built-in Python installations we added direct support for.
python --version
return 0
fi

local conda="${CONDA_EXE-}" # Try to get the activated conda executable
if [ -z "${conda}" ]; then # If no conda is found, try to find it in PATH
conda="$(command -v conda || true)"
fi

if [ ! -x "${conda}" ] || [ "${MINIMAL_INSTALL-}" = 1 ]; then # If no conda is found, install it
local miniconda_dir # Keep directories user-independent, to help with Bazel caching
local miniconda_version="Miniconda3-py311_24.4.0-0"
local miniconda_platform=""
local exe_suffix=".sh"

case "${OSTYPE}" in
linux*)
miniconda_dir="/opt/miniconda"
miniconda_platform=Linux
;;
darwin*)
if [ "$(uname -m)" = "arm64" ]; then
HOSTTYPE="arm64"
miniconda_dir="/opt/homebrew/opt/miniconda"
else
HOSTTYPE="x86_64"
miniconda_dir="/usr/local/opt/miniconda"
fi
miniconda_platform=MacOSX
;;
msys*)
miniconda_dir="${ALLUSERSPROFILE}\Miniconda3" # Avoid spaces; prefer the default path
miniconda_platform=Windows
exe_suffix=".exe"
;;
esac

local miniconda_url="https://repo.continuum.io/miniconda/${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}"
local miniconda_target="${HOME}/${miniconda_url##*/}"
curl -f -s -L -o "${miniconda_target}" "${miniconda_url}"
chmod +x "${miniconda_target}"

case "${OSTYPE}" in
msys*)
# We set /AddToPath=0 because
# (1) it doesn't take care of the current shell, and
# (2) it's consistent with -b in the UNIX installers.
MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" \
/RegisterPython=0 /AddToPath=0 /InstallationType=AllUsers /S /D="${miniconda_dir}"
conda="${miniconda_dir}\Scripts\conda.exe"
;;
*)
if [ "${MINIMAL_INSTALL-}" = 1 ]; then
rm -rf "${miniconda_dir}"
fi
mkdir -p -- "${miniconda_dir}"
# We're forced to pass -b for non-interactive mode.
# Unfortunately it inhibits PATH modifications as a side effect.
"${WORKSPACE_DIR}"/ci/suppress_output "${miniconda_target}" -f -b -p "${miniconda_dir}"
conda="${miniconda_dir}/bin/conda"
;;
esac
fi

if [ ! -x "${CONDA_PYTHON_EXE-}" ]; then # If conda isn't activated, activate it
local restore_shell_state=""
if [ -o xtrace ]; then set +x && restore_shell_state="set -x"; fi # Disable set -x (noisy here)

# TODO(mehrdadn): conda activation is buggy on MSYS2; it adds C:/... to PATH,
# which gets split on a colon. Is it necessary to work around this?
eval "$("${conda}" shell."${SHELL##*/}" hook)" # Activate conda
conda init "${SHELL##*/}" # Add to future shells

${restore_shell_state} # Restore set -x
fi

local python_version
python_version="$(python -s -c "import sys; print('%s.%s' % sys.version_info[:2])")"
if [ -n "${PYTHON-}" ] && [ "${PYTHON}" != "${python_version}" ]; then # Update Python version
(
set +x
echo "Updating Anaconda Python ${python_version} to ${PYTHON}..."
"${WORKSPACE_DIR}"/ci/suppress_output conda install -q -y python="${PYTHON}"
)
elif [ "${MINIMAL_INSTALL-}" = "1" ]; then # Reset environment
(
set +x
echo "Resetting Anaconda Python ${python_version}..."
"${WORKSPACE_DIR}"/ci/suppress_output conda install -q -y --rev 0
)
fi

if [[ "${PYTHON-}" != "3.12" ]]; then
# Install mpi4py as a test dependency for Python <3.12; currently mpi4py is not
# available for Python 3.12
"${WORKSPACE_DIR}"/ci/suppress_output conda install -c anaconda mpi4py -y
fi

command -V python
test -x "${CONDA_PYTHON_EXE}" # make sure conda is activated
}

install_shellcheck() {
local shellcheck_version="0.7.1"
if [ "${shellcheck_version}" != "$(command -v shellcheck > /dev/null && shellcheck --version | sed -n "s/version: //p")" ]; then
Expand Down
115 changes: 115 additions & 0 deletions ci/env/install-miniconda.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
#!/usr/bin/env bash

install_miniconda() {
if [ "${OSTYPE}" = msys ]; then
# Windows is on GitHub Actions, whose built-in Python installations we added direct support for.
python --version
return 0
fi

local conda="${CONDA_EXE-}" # Try to get the activated conda executable
if [ -z "${conda}" ]; then # If no conda is found, try to find it in PATH
conda="$(command -v conda || true)"
fi

if [ ! -x "${conda}" ] || [ "${MINIMAL_INSTALL-}" = 1 ]; then # If no conda is found, install it
local miniconda_dir # Keep directories user-independent, to help with Bazel caching
local miniconda_version="Miniconda3-py311_24.4.0-0"
local miniconda_platform=""
local exe_suffix=".sh"

case "${OSTYPE}" in
linux*)
miniconda_dir="/opt/miniconda"
miniconda_platform=Linux
;;
darwin*)
if [ "$(uname -m)" = "arm64" ]; then
HOSTTYPE="arm64"
miniconda_dir="/opt/homebrew/opt/miniconda"
else
HOSTTYPE="x86_64"
miniconda_dir="/usr/local/opt/miniconda"
fi
miniconda_platform=MacOSX
;;
msys*)
miniconda_dir="${ALLUSERSPROFILE}\Miniconda3" # Avoid spaces; prefer the default path
miniconda_platform=Windows
exe_suffix=".exe"
;;
esac

local miniconda_url="https://repo.continuum.io/miniconda/${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}"
local miniconda_target="${HOME}/${miniconda_url##*/}"
curl -f -s -L -o "${miniconda_target}" "${miniconda_url}"
chmod +x "${miniconda_target}"

case "${OSTYPE}" in
msys*)
# We set /AddToPath=0 because
# (1) it doesn't take care of the current shell, and
# (2) it's consistent with -b in the UNIX installers.
MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" \
/RegisterPython=0 /AddToPath=0 /InstallationType=AllUsers /S /D="${miniconda_dir}"
conda="${miniconda_dir}\Scripts\conda.exe"
;;
*)
if [ "${MINIMAL_INSTALL-}" = 1 ]; then
rm -rf "${miniconda_dir}"
fi
mkdir -p -- "${miniconda_dir}"
# We're forced to pass -b for non-interactive mode.
# Unfortunately it inhibits PATH modifications as a side effect.
"${WORKSPACE_DIR}"/ci/suppress_output "${miniconda_target}" -f -b -p "${miniconda_dir}"
conda="${miniconda_dir}/bin/conda"
;;
esac
fi

if [ ! -x "${CONDA_PYTHON_EXE-}" ]; then # If conda isn't activated, activate it
local restore_shell_state=""
if [ -o xtrace ]; then set +x && restore_shell_state="set -x"; fi # Disable set -x (noisy here)

# TODO(mehrdadn): conda activation is buggy on MSYS2; it adds C:/... to PATH,
# which gets split on a colon. Is it necessary to work around this?
eval "$("${conda}" shell."${SHELL##*/}" hook)" # Activate conda
conda init "${SHELL##*/}" # Add to future shells

${restore_shell_state} # Restore set -x
fi

local python_version
python_version="$(python -s -c "import sys; print('%s.%s' % sys.version_info[:2])")"
if [ -n "${PYTHON-}" ] && [ "${PYTHON}" != "${python_version}" ]; then # Update Python version
(
set +x
echo "Updating Anaconda Python ${python_version} to ${PYTHON}..."
"${WORKSPACE_DIR}"/ci/suppress_output conda install -q -y python="${PYTHON}"
)
elif [ "${MINIMAL_INSTALL-}" = "1" ]; then # Reset environment
(
set +x
echo "Resetting Anaconda Python ${python_version}..."
"${WORKSPACE_DIR}"/ci/suppress_output conda install -q -y --rev 0
)
fi

if [[ "${PYTHON-}" != "3.12" ]]; then
# Install mpi4py as a test dependency for Python <3.12; currently mpi4py is not
# available for Python 3.12
"${WORKSPACE_DIR}"/ci/suppress_output conda install -c anaconda mpi4py -y
fi

command -V python
test -x "${CONDA_PYTHON_EXE}" # make sure conda is activated
}

if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
set -exuo pipefail

SCRIPT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
WORKSPACE_DIR="${SCRIPT_DIR}/../.."
install_miniconda
fi

4 changes: 2 additions & 2 deletions python/ray/air/util/data_batch_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ def _cast_ndarray_columns_to_tensor_extension(df: "pd.DataFrame") -> "pd.DataFra
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=FutureWarning)
warnings.simplefilter("ignore", category=SettingWithCopyWarning)
df.loc[:, col_name] = TensorArray(col)
df[col_name] = TensorArray(col)
except Exception as e:
raise ValueError(
f"Tried to cast column {col_name} to the TensorArray tensor "
Expand Down Expand Up @@ -354,5 +354,5 @@ def _cast_tensor_columns_to_ndarrays(df: "pd.DataFrame") -> "pd.DataFrame":
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=FutureWarning)
warnings.simplefilter("ignore", category=SettingWithCopyWarning)
df.loc[:, col_name] = pd.Series(list(col.to_numpy()))
df[col_name] = list(col.to_numpy())
return df
8 changes: 4 additions & 4 deletions python/ray/data/tests/test_tensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@ def test_tensors_in_tables_pandas_roundtrip(
ds_df = ds.to_pandas()
expected_df = df + 1
if enable_automatic_tensor_extension_cast:
expected_df.loc[:, "two"] = list(expected_df["two"].to_numpy())
expected_df["two"] = list(expected_df["two"].to_numpy())
pd.testing.assert_frame_equal(ds_df, expected_df)


Expand All @@ -585,7 +585,7 @@ def test_tensors_in_tables_pandas_roundtrip_variable_shaped(
ds_df = ds.to_pandas()
expected_df = df + 1
if enable_automatic_tensor_extension_cast:
expected_df.loc[:, "two"] = _create_possibly_ragged_ndarray(
expected_df["two"] = _create_possibly_ragged_ndarray(
expected_df["two"].to_numpy()
)
pd.testing.assert_frame_equal(ds_df, expected_df)
Expand Down Expand Up @@ -873,8 +873,8 @@ def test_tensors_in_tables_iter_batches(
)
df = pd.concat([df1, df2], ignore_index=True)
if enable_automatic_tensor_extension_cast:
df.loc[:, "one"] = list(df["one"].to_numpy())
df.loc[:, "two"] = list(df["two"].to_numpy())
df["one"] = list(df["one"].to_numpy())
df["two"] = list(df["two"].to_numpy())
ds = ray.data.from_pandas([df1, df2])
batches = list(ds.iter_batches(batch_size=2, batch_format="pandas"))
assert len(batches) == 3
Expand Down
6 changes: 6 additions & 0 deletions python/ray/util/dask/tests/test_dask_callback.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
import sys

import dask
import pytest

import ray
from ray.util.dask import ray_dask_get, RayDaskCallback

pytestmark = pytest.mark.skipif(
sys.version_info >= (3, 12), reason="Skip dask tests for Python version 3.12+"
)


@pytest.fixture
def ray_start_1_cpu():
Expand Down
Loading

0 comments on commit 9a840a1

Please sign in to comment.